From 7385efef351dfa14e61af9438923a1907ca484b4 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 1 Feb 2016 16:54:34 -0500 Subject: [PATCH 01/23] Adding some early Notifications stubs * A basic NotificationTemplate model class with early notification type definitions * Initial implementations of the Email, Slack, and Twilio Notification backends using the Django email backend system * Some dependencies thereof --- awx/main/models/__init__.py | 1 + awx/main/models/notifications.py | 37 +++++++++++++++++++ awx/main/notifications/__init__.py | 0 awx/main/notifications/email_backend.py | 11 ++++++ awx/main/notifications/slack_backend.py | 46 ++++++++++++++++++++++++ awx/main/notifications/twilio_backend.py | 42 ++++++++++++++++++++++ requirements/requirements.txt | 2 ++ 7 files changed, 139 insertions(+) create mode 100644 awx/main/models/notifications.py create mode 100644 awx/main/notifications/__init__.py create mode 100644 awx/main/notifications/email_backend.py create mode 100644 awx/main/notifications/slack_backend.py create mode 100644 awx/main/notifications/twilio_backend.py diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 23cf591e6b..4e6d45f18f 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -17,6 +17,7 @@ from awx.main.models.schedules import * # noqa from awx.main.models.activity_stream import * # noqa from awx.main.models.ha import * # noqa from awx.main.models.configuration import * # noqa +from awx.main.models.notifications import * # noqa # Monkeypatch Django serializer to ignore django-taggit fields (which break # the dumpdata command; see https://github.com/alex/django-taggit/issues/155). diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py new file mode 100644 index 0000000000..e2539edc4b --- /dev/null +++ b/awx/main/models/notifications.py @@ -0,0 +1,37 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging + +from django.db import models +from awx.main.models.base import * # noqa +from awx.main.notifications.email_backend import CustomEmailBackend +from awx.main.notifications.slack_backend import SlackBackend +from awx.main.notifications.twilio_backend import TwilioBackend + +# Django-JSONField +from jsonfield import JSONField + +logger = logging.getLogger('awx.main.models.notifications') + +class NotificationTemplate(CommonModel): + + NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), + ('slack', _('Slack'), SlackBackend), + ('twilio', _('Twilio'), TwilioBackend)] + NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES] + CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) + + class Meta: + app_label = 'main' + + notification_type = models.CharField( + max_length = 32, + choices=NOTIFICATION_TYPE_CHOICES, + ) + + notification_configuration = JSONField(blank=False) + + @property + def notification_class(self): + return CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] diff --git a/awx/main/notifications/__init__.py b/awx/main/notifications/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py new file mode 100644 index 0000000000..47c6c6dd00 --- /dev/null +++ b/awx/main/notifications/email_backend.py @@ -0,0 +1,11 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging + +from django.core.mail.backends.smtp import EmailBackend + +class CustomEmailBackend(EmailBackend): + + init_parameters = ("host", "port", "username", "password", + "use_tls", "use_ssl") diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py new file mode 100644 index 0000000000..07c0f2c4fc --- /dev/null +++ b/awx/main/notifications/slack_backend.py @@ -0,0 +1,46 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging +from slackclient import SlackClient + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.slack_backend') + +class SlackBackend(BaseEmailBackend): + + init_parameters = ('token',) + + def __init__(self, token, fail_silently=False, **kwargs): + super(SlackBackend, self).__init__(fail_silently=fail_silently) + self.token = token + self.connection = None + + def open(self): + if self.connection is not None: + return False + self.connection = SlackClient(self.token) + if not self.connection.rtm_connect(): + if not self.fail_silently: + raise Exception("Slack Notification Token is invalid") + return True + + def close(self): + if self.connection is None: + return + self.connection = None + + def send_messages(self, messages): + if self.connection is None: + self.open() + sent_messages = 0 + for m in messages: + try: + self.connection.rtm_send_message(m.to, m.body) + sent_messages += 1 + except Exception as e: + if not self.fail_silently: + raise + logger.error("Exception sending messages: {}".format(e)) + return sent_messages diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py new file mode 100644 index 0000000000..86d6829c09 --- /dev/null +++ b/awx/main/notifications/twilio_backend.py @@ -0,0 +1,42 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging + +from twilio.rest import TwilioRestClient + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.twilio_backend') + +class TwilioBackend(BaseEmailBackend): + + init_parameters = ('account_sid', 'account_token', 'from_phone',) + + def __init__(self, account_sid, account_token, from_phone, fail_silently=False, **kwargs): + super(TwilioBackend, self).__init__(fail_silently=fail_silently) + self.account_sid = account_sid + self.account_token = account_token + self.from_phone = from_phone + + def send_messages(self, messages): + sent_messages = 0 + try: + connection = TwilioRestClient(self.account_sid, self.account_token) + except Exception as e: + if not self.fail_silently: + raise + logger.error("Exception connecting to Twilio: {}".format(e)) + + for m in messages: + try: + connection.messages.create( + to=m.to, + from_=self.from_phone, + body=m.body) + sent_messages += 1 + except Exception as e: + if not self.fail_silently: + raise + logger.error("Exception sending messages: {}".format(e)) + return sent_messages diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 48857bc6d2..73942d9eec 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -114,9 +114,11 @@ requests==2.5.1 requests-oauthlib==0.5.0 simplejson==3.6.0 six==1.9.0 +slackclient==0.16 statsd==3.2.1 stevedore==1.3.0 suds==0.4 +twilio==4.9.1 warlock==1.1.0 wheel==0.24.0 wsgiref==0.1.2 From 805514990b4eb95065407c80311e10641ab9829b Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 1 Feb 2016 16:55:57 -0500 Subject: [PATCH 02/23] Changes to celery tasks to support success signals Linking in a success callback that will be invoked by our UnifiedJobs in the case they terminate normally. This is where we'll hook in the success notification type. --- awx/main/management/commands/run_task_system.py | 7 ++++--- awx/main/models/unified_jobs.py | 6 +++--- awx/main/tasks.py | 9 ++++++++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/awx/main/management/commands/run_task_system.py b/awx/main/management/commands/run_task_system.py index d49dbf1669..5b5dd3bff0 100644 --- a/awx/main/management/commands/run_task_system.py +++ b/awx/main/management/commands/run_task_system.py @@ -15,7 +15,7 @@ from django.core.management.base import NoArgsCommand # AWX from awx.main.models import * # noqa from awx.main.queue import FifoQueue -from awx.main.tasks import handle_work_error +from awx.main.tasks import handle_work_error, handle_work_success from awx.main.utils import get_system_task_capacity # Celery @@ -265,14 +265,15 @@ def process_graph(graph, task_capacity): [{'type': graph.get_node_type(n['node_object']), 'id': n['node_object'].id} for n in node_dependencies] error_handler = handle_work_error.s(subtasks=dependent_nodes) - start_status = node_obj.start(error_callback=error_handler) + success_handler = handle_work_success.s(task_actual={'type': graph.get_node_type(node_obj), + 'id': node_obj.id}) + start_status = node_obj.start(error_callback=error_handler, success_callback=success_handler) if not start_status: node_obj.status = 'failed' if node_obj.job_explanation: node_obj.job_explanation += ' ' node_obj.job_explanation += 'Task failed pre-start check.' node_obj.save() - # TODO: Run error handler continue remaining_volume -= impact running_impact += impact diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 86ab0b3143..cd519af726 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -717,7 +717,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique tasks that might preclude creating one''' return [] - def start(self, error_callback, **kwargs): + def start(self, error_callback, success_callback, **kwargs): ''' Start the task running via Celery. ''' @@ -743,7 +743,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique # if field not in needed]) if 'extra_vars' in kwargs: self.handle_extra_data(kwargs['extra_vars']) - task_class().apply_async((self.pk,), opts, link_error=error_callback) + task_class().apply_async((self.pk,), opts, link_error=error_callback, link=success_callback) return True def signal_start(self, **kwargs): @@ -765,7 +765,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique # Sanity check: If we are running unit tests, then run synchronously. if getattr(settings, 'CELERY_UNIT_TEST', False): - return self.start(None, **kwargs) + return self.start(None, None, **kwargs) # Save the pending status, and inform the SocketIO listener. self.update_fields(start_args=json.dumps(kwargs), status='pending') diff --git a/awx/main/tasks.py b/awx/main/tasks.py index acfe2022ae..478bb6275c 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -52,7 +52,8 @@ from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, from awx.fact.utils.connection import test_mongo_connection __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate', - 'RunAdHocCommand', 'handle_work_error', 'update_inventory_computed_fields'] + 'RunAdHocCommand', 'handle_work_error', 'handle_work_success', + 'update_inventory_computed_fields'] HIDDEN_PASSWORD = '**********' @@ -159,8 +160,14 @@ def mongodb_control(cmd): p = subprocess.Popen('sudo mongod --shutdown -f /etc/mongod.conf', shell=True) p.wait() +@task(bind=True) +def handle_work_success(self, result, task_actual): + # TODO: Perform Notification tasks + pass + @task(bind=True) def handle_work_error(self, task_id, subtasks=None): + # TODO: Perform Notification tasks print('Executing error task id %s, subtasks: %s' % (str(self.request.id), str(subtasks))) first_task = None From 96b0fb168ff4c83477a9247db430610e8b1e10b0 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 4 Feb 2016 11:29:21 -0500 Subject: [PATCH 03/23] Updating makefile migration generator --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index c5735982b9..18e2d951f8 100644 --- a/Makefile +++ b/Makefile @@ -286,7 +286,7 @@ migrate: # Run after making changes to the models to create a new migration. dbchange: - $(PYTHON) manage.py schemamigration main v14_changes --auto + $(PYTHON) manage.py makemigrations # access database shell, asks for password dbshell: From 172207cd4ba129a5d1f4f654d1320bb25c9e1f15 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 4 Feb 2016 11:30:40 -0500 Subject: [PATCH 04/23] Notification endpoints and url expositions Also some changes to the footprint of the notification handler classes --- awx/api/urls.py | 6 ++++++ awx/api/views.py | 13 +++++++++++++ awx/main/access.py | 13 +++++++++++++ awx/main/models/notifications.py | 8 ++++++++ awx/main/notifications/email_backend.py | 9 +++++++-- awx/main/notifications/slack_backend.py | 2 +- awx/main/notifications/twilio_backend.py | 4 +++- 7 files changed, 51 insertions(+), 4 deletions(-) diff --git a/awx/api/urls.py b/awx/api/urls.py index 2b3a93d852..8e48250560 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -209,6 +209,11 @@ system_job_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/cancel/$', 'system_job_cancel'), ) +notification_template_urls = patterns('awx.api.views', + url(r'^$', 'notification_template_list'), + url(r'^(?P[0-9]+)/$', 'notification_template_detail'), +) + schedule_urls = patterns('awx.api.views', url(r'^$', 'schedule_list'), url(r'^(?P[0-9]+)/$', 'schedule_detail'), @@ -257,6 +262,7 @@ v1_urls = patterns('awx.api.views', url(r'^ad_hoc_command_events/', include(ad_hoc_command_event_urls)), url(r'^system_job_templates/', include(system_job_template_urls)), url(r'^system_jobs/', include(system_job_urls)), + url(r'^notification_templates/', include(notification_template_urls)), url(r'^unified_job_templates/$', 'unified_job_template_list'), url(r'^unified_jobs/$', 'unified_job_list'), url(r'^activity_stream/', include(activity_stream_urls)), diff --git a/awx/api/views.py b/awx/api/views.py index 9a41e779ea..72e1fb606e 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -135,6 +135,7 @@ class ApiV1RootView(APIView): data['system_job_templates'] = reverse('api:system_job_template_list') data['system_jobs'] = reverse('api:system_job_list') data['schedules'] = reverse('api:schedule_list') + data['notification_templates'] = reverse('api:notification_template_list') data['unified_job_templates'] = reverse('api:unified_job_template_list') data['unified_jobs'] = reverse('api:unified_job_list') data['activity_stream'] = reverse('api:activity_stream_list') @@ -2919,6 +2920,18 @@ class AdHocCommandStdout(UnifiedJobStdout): model = AdHocCommand new_in_220 = True +class NotificationTemplateList(ListCreateAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + new_in_300 = True + +class NotificationTemplateDetail(RetrieveDestroyAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + new_in_300 = True + class ActivityStreamList(SimpleListAPIView): model = ActivityStream diff --git a/awx/main/access.py b/awx/main/access.py index e17fc59b02..e4ef4653a7 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1484,6 +1484,18 @@ class ScheduleAccess(BaseAccess): else: return False +class NotificationTemplateAccess(BaseAccess): + ''' + I can see/use a notification template if I have permission to + ''' + model = NotificationTemplate + + def get_queryset(self): + qs = self.model.objects.filter(active=True).distinct() + if self.user.is_superuser: + return qs + return qs + class ActivityStreamAccess(BaseAccess): ''' I can see activity stream events only when I have permission on all objects included in the event @@ -1683,3 +1695,4 @@ register_access(UnifiedJob, UnifiedJobAccess) register_access(ActivityStream, ActivityStreamAccess) register_access(CustomInventoryScript, CustomInventoryScriptAccess) register_access(TowerSettings, TowerSettingsAccess) +register_access(NotificationTemplate, NotificationTemplateAccess) diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index e2539edc4b..81c5b31e7f 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -4,6 +4,9 @@ import logging from django.db import models +from django.core.urlresolvers import reverse +from django.utils.translation import ugettext_lazy as _ + from awx.main.models.base import * # noqa from awx.main.notifications.email_backend import CustomEmailBackend from awx.main.notifications.slack_backend import SlackBackend @@ -14,6 +17,8 @@ from jsonfield import JSONField logger = logging.getLogger('awx.main.models.notifications') +__all__ = ['NotificationTemplate'] + class NotificationTemplate(CommonModel): NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), @@ -32,6 +37,9 @@ class NotificationTemplate(CommonModel): notification_configuration = JSONField(blank=False) + def get_absolute_url(self): + return reverse('api:notification_template_detail', args=(self.pk,)) + @property def notification_class(self): return CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py index 47c6c6dd00..db0a8b3c2f 100644 --- a/awx/main/notifications/email_backend.py +++ b/awx/main/notifications/email_backend.py @@ -7,5 +7,10 @@ from django.core.mail.backends.smtp import EmailBackend class CustomEmailBackend(EmailBackend): - init_parameters = ("host", "port", "username", "password", - "use_tls", "use_ssl") + init_parameters = {"host": {"label": "Host", "type": "string"}, + "port": {"label": "Port", "type": "int"}, + "username": {"label": "Username", "type": "string"}, + "password": {"label": "Password", "type": "password"}, + "use_tls": {"label": "Use TLS", "type": "bool"}, + "use_ssl": {"label": "Use SSL", "type": "bool"}} + diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 07c0f2c4fc..84ae60c3cb 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -10,7 +10,7 @@ logger = logging.getLogger('awx.main.notifications.slack_backend') class SlackBackend(BaseEmailBackend): - init_parameters = ('token',) + init_parameters = {"token": {"label": "Token", "type": "password"}} def __init__(self, token, fail_silently=False, **kwargs): super(SlackBackend, self).__init__(fail_silently=fail_silently) diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index 86d6829c09..d0d2fbfe76 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -11,7 +11,9 @@ logger = logging.getLogger('awx.main.notifications.twilio_backend') class TwilioBackend(BaseEmailBackend): - init_parameters = ('account_sid', 'account_token', 'from_phone',) + init_parameters = {"account_sid": {"label": "Account SID", "type": "string"}, + "account_token": {"label": "Account Token", "type": "password"}, + "from_phone": {"label": "Source Phone Number", "type": "string"}} def __init__(self, account_sid, account_token, from_phone, fail_silently=False, **kwargs): super(TwilioBackend, self).__init__(fail_silently=fail_silently) From 319deffc180578cd21ccdcdd39aaf19dfb9350f1 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 4 Feb 2016 11:31:34 -0500 Subject: [PATCH 05/23] Implement notification serializer and validations --- awx/api/metadata.py | 8 +++++++- awx/api/serializers.py | 15 +++++++++++++++ 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index 46ea3f36da..b5e6d7043a 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -12,7 +12,7 @@ from rest_framework import serializers from rest_framework.request import clone_request # Ansible Tower -from awx.main.models import InventorySource +from awx.main.models import InventorySource, NotificationTemplate class Metadata(metadata.SimpleMetadata): @@ -55,6 +55,12 @@ class Metadata(metadata.SimpleMetadata): get_group_by_choices = getattr(InventorySource, 'get_%s_group_by_choices' % cp) field_info['%s_group_by_choices' % cp] = get_group_by_choices() + # Special handling of notification configuration where the required properties + # are conditional on the type selected. + if field.field_name == 'notification_configuration': + for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES: + field_info[notification_type_name] = notification_type_class.init_parameters + # Update type of fields returned... if field.field_name == 'type': field_info['type'] = 'multiple choice' diff --git a/awx/api/serializers.py b/awx/api/serializers.py index f655b35f4e..b513af2c68 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2035,6 +2035,21 @@ class JobLaunchSerializer(BaseSerializer): attrs = super(JobLaunchSerializer, self).validate(attrs) return attrs +class NotificationTemplateSerializer(BaseSerializer): + + class Meta: + model = NotificationTemplate + fields = ('*', 'notification_type', 'notification_configuration') + + def validate(self, attrs): + notification_class = NotificationTemplate.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] + missing_fields = [] + for field in notification_class.init_parameters: + if field not in attrs['notification_configuration']: + missing_fields.append(field) + if missing_fields: + raise serializers.ValidationError("Missing required fields for Notification Configuration: {}".format(missing_fields)) + return attrs class ScheduleSerializer(BaseSerializer): From 8db2f6040534c461bb9ce48662e7b341883f4e23 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 9 Feb 2016 23:12:55 -0500 Subject: [PATCH 06/23] Notification serializers, views, and tasks * Implement concrete Notification model for notification runs * Implement NotificationTemplate and Notification serializers and views * Implement ancillary views * Implement NotificationTemplate trigger m2m fields on all job templates via a fields mixin * Link NotificationTemplates with an org * Link notifications with the activity stream * Implement Notification celery tasks * Extend Backend field parameters to identify sender and receiver as parameters needed by the message and not the backend itself * Updates to backends to better fit the django email backend model as it relates to Messages * Implement success job chain task + notifications * Implement notifications in error job chain task --- awx/api/serializers.py | 45 ++++++- awx/api/urls.py | 24 ++++ awx/api/views.py | 157 ++++++++++++++++++++++- awx/main/access.py | 14 ++ awx/main/models/__init__.py | 2 + awx/main/models/activity_stream.py | 2 + awx/main/models/base.py | 25 +++- awx/main/models/inventory.py | 10 ++ awx/main/models/jobs.py | 11 ++ awx/main/models/notifications.py | 93 +++++++++++++- awx/main/models/organization.py | 2 +- awx/main/models/projects.py | 19 +++ awx/main/models/unified_jobs.py | 16 ++- awx/main/notifications/email_backend.py | 6 +- awx/main/notifications/slack_backend.py | 10 +- awx/main/notifications/twilio_backend.py | 7 +- awx/main/signals.py | 2 + awx/main/tasks.py | 77 ++++++++++- 18 files changed, 502 insertions(+), 20 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index b513af2c68..7bc25f532d 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -457,6 +457,8 @@ class BaseSerializer(serializers.ModelSerializer): ret.pop(parent_key, None) return ret +class EmptySerializer(serializers.Serializer): + pass class BaseFactSerializer(DocumentSerializer): @@ -765,7 +767,11 @@ class OrganizationSerializer(BaseSerializer): users = reverse('api:organization_users_list', args=(obj.pk,)), admins = reverse('api:organization_admins_list', args=(obj.pk,)), teams = reverse('api:organization_teams_list', args=(obj.pk,)), - activity_stream = reverse('api:organization_activity_stream_list', args=(obj.pk,)) + activity_stream = reverse('api:organization_activity_stream_list', args=(obj.pk,)), + notifiers = reverse('api:organization_notifiers_list', args=(obj.pk,)), + notifiers_any = reverse('api:organization_notifications_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:organization_notifications_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:organization_notifications_error_list', args=(obj.pk,)), )) return res @@ -845,6 +851,9 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): project_updates = reverse('api:project_updates_list', args=(obj.pk,)), schedules = reverse('api:project_schedules_list', args=(obj.pk,)), activity_stream = reverse('api:project_activity_stream_list', args=(obj.pk,)), + notifiers_any = reverse('api:project_notifications_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:project_notifications_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:project_notifications_error_list', args=(obj.pk,)), )) # Backwards compatibility. if obj.current_update: @@ -888,6 +897,7 @@ class ProjectUpdateSerializer(UnifiedJobSerializer, ProjectOptionsSerializer): res.update(dict( project = reverse('api:project_detail', args=(obj.project.pk,)), cancel = reverse('api:project_update_cancel', args=(obj.pk,)), + notifications = reverse('api:project_update_notifications_list', args=(obj.pk,)), )) return res @@ -1288,6 +1298,9 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt activity_stream = reverse('api:inventory_activity_stream_list', args=(obj.pk,)), hosts = reverse('api:inventory_source_hosts_list', args=(obj.pk,)), groups = reverse('api:inventory_source_groups_list', args=(obj.pk,)), + notifiers_any = reverse('api:inventory_source_notifications_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:inventory_source_notifications_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:inventory_source_notifications_error_list', args=(obj.pk,)), )) if obj.inventory and obj.inventory.active: res['inventory'] = reverse('api:inventory_detail', args=(obj.inventory.pk,)) @@ -1332,6 +1345,7 @@ class InventoryUpdateSerializer(UnifiedJobSerializer, InventorySourceOptionsSeri res.update(dict( inventory_source = reverse('api:inventory_source_detail', args=(obj.inventory_source.pk,)), cancel = reverse('api:inventory_update_cancel', args=(obj.pk,)), + notifications = reverse('api:inventory_update_notifications_list', args=(obj.pk,)), )) return res @@ -1550,6 +1564,9 @@ class JobTemplateSerializer(UnifiedJobTemplateSerializer, JobOptionsSerializer): schedules = reverse('api:job_template_schedules_list', args=(obj.pk,)), activity_stream = reverse('api:job_template_activity_stream_list', args=(obj.pk,)), launch = reverse('api:job_template_launch', args=(obj.pk,)), + notifiers_any = reverse('api:job_template_notifications_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:job_template_notifications_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:job_template_notifications_error_list', args=(obj.pk,)), )) if obj.host_config_key: res['callback'] = reverse('api:job_template_callback', args=(obj.pk,)) @@ -1604,6 +1621,7 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer): job_tasks = reverse('api:job_job_tasks_list', args=(obj.pk,)), job_host_summaries = reverse('api:job_job_host_summaries_list', args=(obj.pk,)), activity_stream = reverse('api:job_activity_stream_list', args=(obj.pk,)), + notifications = reverse('api:job_notifications_list', args=(obj.pk,)), )) if obj.job_template and obj.job_template.active: res['job_template'] = reverse('api:job_template_detail', @@ -2039,7 +2057,15 @@ class NotificationTemplateSerializer(BaseSerializer): class Meta: model = NotificationTemplate - fields = ('*', 'notification_type', 'notification_configuration') + fields = ('*', 'organization', 'notification_type', 'notification_configuration') + + def get_related(self, obj): + res = super(NotificationTemplateSerializer, self).get_related(obj) + res.update(dict( + test = reverse('api:notification_template_test', args=(obj.pk,)), + notifications = reverse('api:notification_template_notification_list', args=(obj.pk,)), + )) + return res def validate(self, attrs): notification_class = NotificationTemplate.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] @@ -2047,10 +2073,25 @@ class NotificationTemplateSerializer(BaseSerializer): for field in notification_class.init_parameters: if field not in attrs['notification_configuration']: missing_fields.append(field) + # TODO: Type checks if missing_fields: raise serializers.ValidationError("Missing required fields for Notification Configuration: {}".format(missing_fields)) return attrs +class NotificationSerializer(BaseSerializer): + + class Meta: + model = Notification + fields = ('*', '-name', '-description', 'notifier', 'error', 'status', 'notifications_sent', + 'notification_type', 'recipients', 'subject', 'body') + + def get_related(self, obj): + res = super(NotificationSerializer, self).get_related(obj) + res.update(dict( + notification_template = reverse('api:notification_template_detail', args=(obj.notifier.pk,)), + )) + return res + class ScheduleSerializer(BaseSerializer): class Meta: diff --git a/awx/api/urls.py b/awx/api/urls.py index 8e48250560..7e55e46a5c 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -20,6 +20,10 @@ organization_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/projects/$', 'organization_projects_list'), url(r'^(?P[0-9]+)/teams/$', 'organization_teams_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'organization_activity_stream_list'), + url(r'^(?P[0-9]+)/notifiers/$', 'organization_notifiers_list'), + url(r'^(?P[0-9]+)/notifications_any/$', 'organization_notifications_any_list'), + url(r'^(?P[0-9]+)/notifications_error/$', 'organization_notifications_error_list'), + url(r'^(?P[0-9]+)/notifications_success/$', 'organization_notifications_success_list'), ) user_urls = patterns('awx.api.views', @@ -44,12 +48,16 @@ project_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/project_updates/$', 'project_updates_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'project_activity_stream_list'), url(r'^(?P[0-9]+)/schedules/$', 'project_schedules_list'), + url(r'^(?P[0-9]+)/notifications_any/$', 'project_notifications_any_list'), + url(r'^(?P[0-9]+)/notifications_error/$', 'project_notifications_error_list'), + url(r'^(?P[0-9]+)/notifications_success/$', 'project_notifications_success_list'), ) project_update_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/$', 'project_update_detail'), url(r'^(?P[0-9]+)/cancel/$', 'project_update_cancel'), url(r'^(?P[0-9]+)/stdout/$', 'project_update_stdout'), + url(r'^(?P[0-9]+)/notifications/$', 'project_update_notifications_list'), ) team_urls = patterns('awx.api.views', @@ -120,12 +128,16 @@ inventory_source_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/schedules/$', 'inventory_source_schedules_list'), url(r'^(?P[0-9]+)/groups/$', 'inventory_source_groups_list'), url(r'^(?P[0-9]+)/hosts/$', 'inventory_source_hosts_list'), + url(r'^(?P[0-9]+)/notifications_any/$', 'inventory_source_notifications_any_list'), + url(r'^(?P[0-9]+)/notifications_error/$', 'inventory_source_notifications_error_list'), + url(r'^(?P[0-9]+)/notifications_success/$', 'inventory_source_notifications_success_list'), ) inventory_update_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/$', 'inventory_update_detail'), url(r'^(?P[0-9]+)/cancel/$', 'inventory_update_cancel'), url(r'^(?P[0-9]+)/stdout/$', 'inventory_update_stdout'), + url(r'^(?P[0-9]+)/notifications/$', 'inventory_update_notifications_list'), ) inventory_script_urls = patterns('awx.api.views', @@ -153,6 +165,9 @@ job_template_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/schedules/$', 'job_template_schedules_list'), url(r'^(?P[0-9]+)/survey_spec/$', 'job_template_survey_spec'), url(r'^(?P[0-9]+)/activity_stream/$', 'job_template_activity_stream_list'), + url(r'^(?P[0-9]+)/notifications_any/$', 'job_template_notifications_any_list'), + url(r'^(?P[0-9]+)/notifications_error/$', 'job_template_notifications_error_list'), + url(r'^(?P[0-9]+)/notifications_success/$', 'job_template_notifications_success_list'), ) job_urls = patterns('awx.api.views', @@ -167,6 +182,7 @@ job_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/job_tasks/$', 'job_job_tasks_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'job_activity_stream_list'), url(r'^(?P[0-9]+)/stdout/$', 'job_stdout'), + url(r'^(?P[0-9]+)/notifications/$', 'job_notifications_list'), ) job_host_summary_urls = patterns('awx.api.views', @@ -212,6 +228,13 @@ system_job_urls = patterns('awx.api.views', notification_template_urls = patterns('awx.api.views', url(r'^$', 'notification_template_list'), url(r'^(?P[0-9]+)/$', 'notification_template_detail'), + url(r'^(?P[0-9]+)/test/$', 'notification_template_test'), + url(r'^(?P[0-9]+)/notifications/$', 'notification_template_notification_list'), +) + +notification_urls = patterns('awx.api.views', + url(r'^$', 'notification_list'), + url(r'^(?P[0-9]+)/$', 'notification_detail'), ) schedule_urls = patterns('awx.api.views', @@ -263,6 +286,7 @@ v1_urls = patterns('awx.api.views', url(r'^system_job_templates/', include(system_job_template_urls)), url(r'^system_jobs/', include(system_job_urls)), url(r'^notification_templates/', include(notification_template_urls)), + url(r'^notifications/', include(notification_urls)), url(r'^unified_job_templates/$', 'unified_job_template_list'), url(r'^unified_jobs/$', 'unified_job_list'), url(r'^activity_stream/', include(activity_stream_urls)), diff --git a/awx/api/views.py b/awx/api/views.py index 72e1fb606e..be1bd3b609 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -56,7 +56,7 @@ from social.backends.utils import load_backends # AWX from awx.main.task_engine import TaskSerializer, TASK_FILE, TEMPORARY_TASK_FILE -from awx.main.tasks import mongodb_control +from awx.main.tasks import mongodb_control, send_notifications from awx.main.access import get_user_queryset from awx.main.ha import is_ha_environment from awx.api.authentication import TaskAuthentication, TokenGetAuthentication @@ -136,6 +136,7 @@ class ApiV1RootView(APIView): data['system_jobs'] = reverse('api:system_job_list') data['schedules'] = reverse('api:schedule_list') data['notification_templates'] = reverse('api:notification_template_list') + data['notifications'] = reverse('api:notification_list') data['unified_job_templates'] = reverse('api:unified_job_template_list') data['unified_jobs'] = reverse('api:unified_job_list') data['activity_stream'] = reverse('api:activity_stream_list') @@ -684,6 +685,35 @@ class OrganizationActivityStreamList(SubListAPIView): # Okay, let it through. return super(type(self), self).get(request, *args, **kwargs) +class OrganizationNotifiersList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Organization + relationship = 'notification_templates' + parent_key = 'organization' + +class OrganizationNotificationsAnyList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Organization + relationship = 'notification_any' + +class OrganizationNotificationsErrorList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Organization + relationship = 'notification_erros' + +class OrganizationNotificationsSuccessList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Organization + relationship = 'notification_success' + class TeamList(ListCreateAPIView): model = Team @@ -849,6 +879,26 @@ class ProjectActivityStreamList(SubListAPIView): return qs.filter(project=parent) return qs.filter(Q(project=parent) | Q(credential__in=parent.credential)) +class ProjectNotificationsAnyList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Project + relationship = 'notification_any' + +class ProjectNotificationsErrorList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Project + relationship = 'notification_errors' + +class ProjectNotificationsSuccessList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = Project + relationship = 'notification_success' class ProjectUpdatesList(SubListAPIView): @@ -899,6 +949,12 @@ class ProjectUpdateCancel(RetrieveAPIView): else: return self.http_method_not_allowed(request, *args, **kwargs) +class ProjectUpdateNotificationsList(SubListAPIView): + + model = Notification + serializer_class = NotificationSerializer + parent_model = Project + relationship = 'notifications' class UserList(ListCreateAPIView): @@ -1725,6 +1781,27 @@ class InventorySourceActivityStreamList(SubListAPIView): # Okay, let it through. return super(type(self), self).get(request, *args, **kwargs) +class InventorySourceNotificationsAnyList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = InventorySource + relationship = 'notification_any' + +class InventorySourceNotificationsErrorList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = InventorySource + relationship = 'notification_errors' + +class InventorySourceNotificationsSuccessList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = InventorySource + relationship = 'notification_success' + class InventorySourceHostsList(SubListAPIView): model = Host @@ -1789,6 +1866,13 @@ class InventoryUpdateCancel(RetrieveAPIView): else: return self.http_method_not_allowed(request, *args, **kwargs) +class InventoryUpdateNotificationsList(SubListAPIView): + + model = Notification + serializer_class = NotificationSerializer + parent_model = InventoryUpdate + relationship = 'notifications' + class JobTemplateList(ListCreateAPIView): model = JobTemplate @@ -1943,6 +2027,27 @@ class JobTemplateActivityStreamList(SubListAPIView): # Okay, let it through. return super(type(self), self).get(request, *args, **kwargs) +class JobTemplateNotificationsAnyList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = JobTemplate + relationship = 'notification_any' + +class JobTemplateNotificationsErrorList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = JobTemplate + relationship = 'notification_errors' + +class JobTemplateNotificationsSuccessList(SubListCreateAttachDetachAPIView): + + model = NotificationTemplate + serializer_class = NotificationTemplateSerializer + parent_model = JobTemplate + relationship = 'notification_success' + class JobTemplateCallback(GenericAPIView): model = JobTemplate @@ -2129,7 +2234,7 @@ class SystemJobTemplateDetail(RetrieveAPIView): class SystemJobTemplateLaunch(GenericAPIView): model = SystemJobTemplate - # FIXME: Add serializer class to define fields in OPTIONS request! + serializer_class = EmptySerializer def get(self, request, *args, **kwargs): return Response({}) @@ -2276,6 +2381,13 @@ class JobRelaunch(RetrieveAPIView, GenericAPIView): headers = {'Location': new_job.get_absolute_url()} return Response(data, status=status.HTTP_201_CREATED, headers=headers) +class JobNotificationsList(SubListAPIView): + + model = Notification + serializer_class = NotificationSerializer + parent_model = Job + relationship = 'notifications' + class BaseJobHostSummariesList(SubListAPIView): model = JobHostSummary @@ -2926,12 +3038,51 @@ class NotificationTemplateList(ListCreateAPIView): serializer_class = NotificationTemplateSerializer new_in_300 = True -class NotificationTemplateDetail(RetrieveDestroyAPIView): +class NotificationTemplateDetail(RetrieveUpdateDestroyAPIView): model = NotificationTemplate serializer_class = NotificationTemplateSerializer new_in_300 = True +class NotificationTemplateTest(GenericAPIView): + + view_name = 'Notification Template Test' + model = NotificationTemplate + serializer_class = EmptySerializer + new_in_300 = True + + def post(self, request, *args, **kwargs): + obj = self.get_object() + notification = obj.generate_notification("Tower Notification Test", "Ansible Tower Test Notification") + if not notification: + return Response({}, status=status.HTTP_400_BAD_REQUEST) + else: + send_notifications.delay([notification.id]) + headers = {'Location': notification.get_absolute_url()} + return Response({"notification": notification.id}, + headers=headers, + status=status.HTTP_202_ACCEPTED) + +class NotificationTemplateNotificationList(SubListAPIView): + + model = Notification + serializer_class = NotificationSerializer + parent_model = NotificationTemplate + relationship = 'notifications' + parent_key = 'notifier' + +class NotificationList(ListAPIView): + + model = Notification + serializer_class = NotificationSerializer + new_in_300 = True + +class NotificationDetail(RetrieveAPIView): + + model = NotificationTemplate + serializer_class = NotificationSerializer + new_in_300 = True + class ActivityStreamList(SimpleListAPIView): model = ActivityStream diff --git a/awx/main/access.py b/awx/main/access.py index e4ef4653a7..3ffbaf7f85 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1496,6 +1496,19 @@ class NotificationTemplateAccess(BaseAccess): return qs return qs +class NotificationAccess(BaseAccess): + ''' + I can see/use a notification if I have permission to + ''' + model = Notification + + def get_queryset(self): + qs = self.model.objects.distinct() + if self.user.is_superuser: + return qs + return qs + + class ActivityStreamAccess(BaseAccess): ''' I can see activity stream events only when I have permission on all objects included in the event @@ -1696,3 +1709,4 @@ register_access(ActivityStream, ActivityStreamAccess) register_access(CustomInventoryScript, CustomInventoryScriptAccess) register_access(TowerSettings, TowerSettingsAccess) register_access(NotificationTemplate, NotificationTemplateAccess) +register_access(Notification, NotificationAccess) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 4e6d45f18f..2397b6137b 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -61,3 +61,5 @@ activity_stream_registrar.connect(AdHocCommand) activity_stream_registrar.connect(Schedule) activity_stream_registrar.connect(CustomInventoryScript) activity_stream_registrar.connect(TowerSettings) +activity_stream_registrar.connect(NotificationTemplate) +activity_stream_registrar.connect(Notification) diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index b695831ada..12a54c7af2 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -53,6 +53,8 @@ class ActivityStream(models.Model): ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) + notification_template = models.ManyToManyField("NotificationTemplate", blank=True) + notification = models.ManyToManyField("Notification", blank=True) def get_absolute_url(self): return reverse('api:activity_stream_detail', args=(self.pk,)) diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 61515d7d18..f3f158855c 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -25,7 +25,7 @@ from awx.main.utils import encrypt_field __all__ = ['VarsDictProperty', 'BaseModel', 'CreatedModifiedModel', 'PasswordFieldsModel', 'PrimordialModel', 'CommonModel', - 'CommonModelNameNotUnique', + 'CommonModelNameNotUnique', 'NotificationFieldsModel', 'PERM_INVENTORY_ADMIN', 'PERM_INVENTORY_READ', 'PERM_INVENTORY_WRITE', 'PERM_INVENTORY_DEPLOY', 'PERM_INVENTORY_SCAN', 'PERM_INVENTORY_CHECK', 'PERM_JOBTEMPLATE_CREATE', 'JOB_TYPE_CHOICES', @@ -337,3 +337,26 @@ class CommonModelNameNotUnique(PrimordialModel): max_length=512, unique=False, ) + +class NotificationFieldsModel(BaseModel): + + class Meta: + abstract = True + + notification_errors = models.ManyToManyField( + "NotificationTemplate", + blank=True, + related_name='%(class)s_notifications_for_errors' + ) + + notification_success = models.ManyToManyField( + "NotificationTemplate", + blank=True, + related_name='%(class)s_notifications_for_success' + ) + + notification_any = models.ManyToManyField( + "NotificationTemplate", + blank=True, + related_name='%(class)s_notifications_for_any' + ) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 37b1dafc4b..febf010f20 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -23,6 +23,7 @@ from awx.main.managers import HostManager from awx.main.models.base import * # noqa from awx.main.models.jobs import Job from awx.main.models.unified_jobs import * # noqa +from awx.main.models.notifications import NotificationTemplate from awx.main.utils import ignore_inventory_computed_fields, _inventory_updates __all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript'] @@ -1180,6 +1181,15 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): return True return False + @property + def notifiers(self): + # Return all notifiers defined on the Project, and on the Organization for each trigger type + base_notifiers = NotificationTemplate.objects.filter(active=True) + error_notifiers = list(base_notifiers.filter(organization_notifications_for_errors__in=self)) + success_notifiers = list(base_notifiers.filter(organization_notifications_for_success__in=self)) + any_notifiers = list(base_notifiers.filter(organization_notifications_for_any__in=self)) + return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) + def clean_source(self): source = self.source if source and self.group: diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 833d20a9b4..42f7ccf676 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -22,6 +22,7 @@ from jsonfield import JSONField from awx.main.constants import CLOUD_PROVIDERS from awx.main.models.base import * # noqa from awx.main.models.unified_jobs import * # noqa +from awx.main.models.notifications import NotificationTemplate from awx.main.utils import decrypt_field, ignore_inventory_computed_fields from awx.main.utils import emit_websocket_notification from awx.main.redact import PlainTextCleaner @@ -330,6 +331,16 @@ class JobTemplate(UnifiedJobTemplate, JobOptions): def _can_update(self): return self.can_start_without_user_input() + @property + def notifiers(self): + # Return all notifiers defined on the Job Template, on the Project, and on the Organization for each trigger type + # TODO: Currently there is no org fk on project so this will need to be added once that is + # available after the rbac pr + base_notifiers = NotificationTemplate.objects.filter(active=True) + error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_errors__in=[self, self.project])) + success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_success__in=[self, self.project])) + any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_any__in=[self, self.project])) + return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) class Job(UnifiedJob, JobOptions): ''' diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 81c5b31e7f..a89f460e64 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -5,7 +5,9 @@ import logging from django.db import models from django.core.urlresolvers import reverse +from django.core.mail.message import EmailMessage from django.utils.translation import ugettext_lazy as _ +from django.utils.encoding import smart_str from awx.main.models.base import * # noqa from awx.main.notifications.email_backend import CustomEmailBackend @@ -17,7 +19,7 @@ from jsonfield import JSONField logger = logging.getLogger('awx.main.models.notifications') -__all__ = ['NotificationTemplate'] +__all__ = ['NotificationTemplate', 'Notification'] class NotificationTemplate(CommonModel): @@ -30,6 +32,14 @@ class NotificationTemplate(CommonModel): class Meta: app_label = 'main' + organization = models.ForeignKey( + 'Organization', + blank=False, + null=True, + on_delete=models.SET_NULL, + related_name='notification_templates', + ) + notification_type = models.CharField( max_length = 32, choices=NOTIFICATION_TYPE_CHOICES, @@ -42,4 +52,83 @@ class NotificationTemplate(CommonModel): @property def notification_class(self): - return CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] + return self.CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] + + @property + def recipients(self): + return self.notification_configuration[self.notification_class.recipient_parameter] + + def generate_notification(self, subject, message): + notification = Notification(notifier=self, + notification_type=self.notification_type, + recipients=smart_str(self.recipients), + subject=subject, + body=message) + notification.save() + return notification + + def send(self, subject, body): + recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter) + sender = self.notification_configuration.pop(self.notification_class.sender_parameter, None) + backend_obj = self.notification_class(**self.notification_configuration) + notification_obj = EmailMessage(subject, body, sender, recipients) + return backend_obj.send_messages([notification_obj]) + +class Notification(CreatedModifiedModel): + ''' + A notification event emitted when a Notifier is run + ''' + + NOTIFICATION_STATE_CHOICES = [ + ('pending', _('Pending')), + ('successful', _('Successful')), + ('failed', _('Failed')), + ] + + class Meta: + app_label = 'main' + ordering = ('pk',) + + notifier = models.ForeignKey( + 'NotificationTemplate', + related_name='notifications', + on_delete=models.CASCADE, + editable=False + ) + status = models.CharField( + max_length=20, + choices=NOTIFICATION_STATE_CHOICES, + default='pending', + editable=False, + ) + error = models.TextField( + blank=True, + default='', + editable=False, + ) + notifications_sent = models.IntegerField( + default=0, + editable=False, + ) + notification_type = models.CharField( + max_length = 32, + choices=NotificationTemplate.NOTIFICATION_TYPE_CHOICES, + ) + recipients = models.TextField( + blank=True, + default='', + editable=False, + ) + subject = models.TextField( + blank=True, + default='', + editable=False, + ) + body = models.TextField( + blank=True, + default='', + editable=False, + ) + + def get_absolute_url(self): + return reverse('api:notification_detail', args=(self.pk,)) diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index c22b907082..58f563735b 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -23,7 +23,7 @@ from awx.main.conf import tower_settings __all__ = ['Organization', 'Team', 'Permission', 'Profile', 'AuthToken'] -class Organization(CommonModel): +class Organization(CommonModel, NotificationFieldsModel): ''' An organization is the basic unit of multi-tenancy divisions ''' diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 2fa6512ca0..730604d3e4 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -10,6 +10,7 @@ import urlparse # Django from django.conf import settings from django.db import models +from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_str, smart_text from django.core.exceptions import ValidationError @@ -20,6 +21,7 @@ from django.utils.timezone import now, make_aware, get_default_timezone from awx.lib.compat import slugify from awx.main.models.base import * # noqa from awx.main.models.jobs import Job +from awx.main.models.notifications import NotificationTemplate from awx.main.models.unified_jobs import * # noqa from awx.main.utils import update_scm_url @@ -309,6 +311,23 @@ class Project(UnifiedJobTemplate, ProjectOptions): return True return False + @property + def notifiers(self): + # Return all notifiers defined on the Project, and on the Organization for each trigger type + # TODO: Currently there is no org fk on project so this will need to be added back once that is + # available after the rbac pr + base_notifiers = NotificationTemplate.objects.filter(active=True) + # error_notifiers = list(base_notifiers.filter(Q(project_notifications_for_errors__in=self) | + # Q(organization_notifications_for_errors__in=self.organization))) + # success_notifiers = list(base_notifiers.filter(Q(project_notifications_for_success__in=self) | + # Q(organization_notifications_for_success__in=self.organization))) + # any_notifiers = list(base_notifiers.filter(Q(project_notifications_for_any__in=self) | + # Q(organization_notifications_for_any__in=self.organization))) + error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_errors=self)) + success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_success=self)) + any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_any=self)) + return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) + def get_absolute_url(self): return reverse('api:project_detail', args=(self.pk,)) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index cd519af726..c6ea2b082b 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -30,6 +30,7 @@ from djcelery.models import TaskMeta # AWX from awx.main.models.base import * # noqa from awx.main.models.schedules import Schedule +from awx.main.models.notifications import Notification from awx.main.utils import decrypt_field, emit_websocket_notification, _inventory_updates from awx.main.redact import UriCleaner @@ -40,7 +41,7 @@ logger = logging.getLogger('awx.main.models.unified_jobs') CAN_CANCEL = ('new', 'pending', 'waiting', 'running') -class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique): +class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, NotificationFieldsModel): ''' Concrete base class for unified job templates. ''' @@ -297,6 +298,14 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique): ''' return kwargs # Override if needed in subclass. + @property + def notifiers(self): + ''' + Return notifiers relevant to this Unified Job Template + ''' + # NOTE: Derived classes should implement + return NotificationTemplate.objects.none() + def create_unified_job(self, **kwargs): ''' Create a new unified job based on this unified job template. @@ -385,6 +394,11 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique editable=False, related_name='%(class)s_blocked_jobs+', ) + notifications = models.ManyToManyField( + 'Notification', + editable=False, + related_name='%(class)s_notifications', + ) cancel_flag = models.BooleanField( blank=True, default=False, diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py index db0a8b3c2f..271f585d5c 100644 --- a/awx/main/notifications/email_backend.py +++ b/awx/main/notifications/email_backend.py @@ -12,5 +12,9 @@ class CustomEmailBackend(EmailBackend): "username": {"label": "Username", "type": "string"}, "password": {"label": "Password", "type": "password"}, "use_tls": {"label": "Use TLS", "type": "bool"}, - "use_ssl": {"label": "Use SSL", "type": "bool"}} + "use_ssl": {"label": "Use SSL", "type": "bool"}, + "sender": {"label": "Sender Email", "type": "string"}, + "recipients": {"label": "Recipient List", "type": "list"}} + recipient_parameter = "recipients" + sender_parameter = "sender" diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 84ae60c3cb..950d5c2c6e 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -10,7 +10,10 @@ logger = logging.getLogger('awx.main.notifications.slack_backend') class SlackBackend(BaseEmailBackend): - init_parameters = {"token": {"label": "Token", "type": "password"}} + init_parameters = {"token": {"label": "Token", "type": "password"}, + "channels": {"label": "Destination Channels", "type": "list"}} + recipient_parameter = "channels" + sender_parameter = None def __init__(self, token, fail_silently=False, **kwargs): super(SlackBackend, self).__init__(fail_silently=fail_silently) @@ -37,8 +40,9 @@ class SlackBackend(BaseEmailBackend): sent_messages = 0 for m in messages: try: - self.connection.rtm_send_message(m.to, m.body) - sent_messages += 1 + for r in m.recipients(): + self.connection.rtm_send_message(r, m.body) + sent_messages += 1 except Exception as e: if not self.fail_silently: raise diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index d0d2fbfe76..cf2ced368b 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -13,7 +13,10 @@ class TwilioBackend(BaseEmailBackend): init_parameters = {"account_sid": {"label": "Account SID", "type": "string"}, "account_token": {"label": "Account Token", "type": "password"}, - "from_phone": {"label": "Source Phone Number", "type": "string"}} + "from_number": {"label": "Source Phone Number", "type": "string"}, + "to_numbers": {"label": "Destination SMS Numbers", "type": "list"}} + recipient_parameter = "to_numbers" + sender_parameter = "from_number" def __init__(self, account_sid, account_token, from_phone, fail_silently=False, **kwargs): super(TwilioBackend, self).__init__(fail_silently=fail_silently) @@ -34,7 +37,7 @@ class TwilioBackend(BaseEmailBackend): try: connection.messages.create( to=m.to, - from_=self.from_phone, + from_=m.from_email, body=m.body) sent_messages += 1 except Exception as e: diff --git a/awx/main/signals.py b/awx/main/signals.py index 8b0c22ec9d..f4d0014905 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -307,6 +307,8 @@ model_serializer_mapping = { Job: JobSerializer, AdHocCommand: AdHocCommandSerializer, TowerSettings: TowerSettingsSerializer, + NotificationTemplate: NotificationTemplateSerializer, + Notification: NotificationSerializer, } def activity_stream_create(sender, instance, created, **kwargs): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 478bb6275c..aff9a6a585 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -53,7 +53,7 @@ from awx.fact.utils.connection import test_mongo_connection __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate', 'RunAdHocCommand', 'handle_work_error', 'handle_work_success', - 'update_inventory_computed_fields'] + 'update_inventory_computed_fields', 'send_notifications'] HIDDEN_PASSWORD = '**********' @@ -65,6 +65,26 @@ Try upgrading OpenSSH or providing your private key in an different format. \ logger = logging.getLogger('awx.main.tasks') +@task() +def send_notifications(notification_list, job_id=None): + if not isinstance(notification_list, list): + raise TypeError("notification_list should be of type list") + for notification_id in notification_list: + notification = Notification.objects.get(id=notification_id) + try: + sent = notification.notifier.send(notification.subject, notification.body) + notification.status = "successful" + notification.notifications_sent = sent + except Exception as e: + logger.error("Send Notification Failed {}".format(e)) + notification.status = "failed" + notification.error = str(e) + finally: + notification.save() + if job_id is not None: + j = UnifiedJob.objects.get(id=job_id) + j.notifications.add(notification) + @task() def bulk_inventory_element_delete(inventory, hosts=[], groups=[]): from awx.main.signals import disable_activity_stream @@ -162,12 +182,41 @@ def mongodb_control(cmd): @task(bind=True) def handle_work_success(self, result, task_actual): - # TODO: Perform Notification tasks - pass + if task_actual['type'] == 'project_update': + instance = ProjectUpdate.objects.get(id=task_actual['id']) + instance_name = instance.name + notifiers = instance.project.notifiers + friendly_name = "Project Update" + elif task_actual['type'] == 'inventory_update': + instance = InventoryUpdate.objects.get(id=task_actual['id']) + instance_name = instance.name + notifiers = instance.inventory_source.notifiers + friendly_name = "Inventory Update" + elif task_actual['type'] == 'job': + instance = Job.objects.get(id=task_actual['id']) + instance_name = instance.job_template.name + notifiers = instance.job_template.notifiers + friendly_name = "Job" + elif task_actual['type'] == 'ad_hoc_command': + instance = AdHocCommand.objects.get(id=task_actual['id']) + instance_name = instance.module_name + notifiers = [] # TODO: Ad-hoc commands need to notify someone + friendly_name = "AdHoc Command" + else: + return + notification_subject = "{} #{} '{}' succeeded on Ansible Tower".format(friendly_name, + task_actual['id'], + instance_name) + notification_body = "{} #{} '{}' succeeded on Ansible Tower\nTo view the output: {}".format(friendly_name, + task_actual['id'], + instance_name, + instance.get_absolute_url()) + send_notifications.delay([n.generate_notification(notification_subject, notification_body) + for n in notifiers.get('success', []) + notifiers.get('any', [])], + job_id=task_actual['id']) @task(bind=True) def handle_work_error(self, task_id, subtasks=None): - # TODO: Perform Notification tasks print('Executing error task id %s, subtasks: %s' % (str(self.request.id), str(subtasks))) first_task = None @@ -180,15 +229,23 @@ def handle_work_error(self, task_id, subtasks=None): if each_task['type'] == 'project_update': instance = ProjectUpdate.objects.get(id=each_task['id']) instance_name = instance.name + notifiers = instance.project.notifiers + friendly_name = "Project Update" elif each_task['type'] == 'inventory_update': instance = InventoryUpdate.objects.get(id=each_task['id']) instance_name = instance.name + notifiers = instance.inventory_source.notifiers + friendly_name = "Inventory Update" elif each_task['type'] == 'job': instance = Job.objects.get(id=each_task['id']) instance_name = instance.job_template.name + notifiers = instance.job_template.notifiers + friendly_name = "Job" elif each_task['type'] == 'ad_hoc_command': instance = AdHocCommand.objects.get(id=each_task['id']) instance_name = instance.module_name + notifiers = [] + friendly_name = "AdHoc Command" else: # Unknown task type break @@ -197,6 +254,7 @@ def handle_work_error(self, task_id, subtasks=None): first_task_id = instance.id first_task_type = each_task['type'] first_task_name = instance_name + first_task_friendly_name = friendly_name if instance.celery_task_id != task_id: instance.status = 'failed' instance.failed = True @@ -204,6 +262,17 @@ def handle_work_error(self, task_id, subtasks=None): (first_task_type, first_task_name, first_task_id) instance.save() instance.socketio_emit_status("failed") + notification_subject = "{} #{} '{}' failed on Ansible Tower".format(first_task_friendly_name, + first_task_id, + first_task_name) + notification_body = "{} #{} '{}' failed on Ansible Tower\nTo view the output: {}".format(first_task_friendly_name, + first_task_id, + first_task_name, + first_task.get_absolute_url()) + send_notifications.delay([n.generate_notification(notification_subject, notification_body).id + for n in notifiers.get('error', []) + notifiers.get('any', [])], + job_id=first_task_id) + @task() def update_inventory_computed_fields(inventory_id, should_update_hosts=True): From 34ebe0a8484fe5eeb897d36d5f8deee6d5779f2f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 10 Feb 2016 15:17:11 -0500 Subject: [PATCH 07/23] Pagerduty and Hipchat backends plus some cleanup --- awx/main/models/notifications.py | 8 +++- awx/main/notifications/hipchat_backend.py | 46 +++++++++++++++++++++ awx/main/notifications/pagerduty_backend.py | 44 ++++++++++++++++++++ awx/main/notifications/twilio_backend.py | 2 +- requirements/requirements.txt | 1 + 5 files changed, 99 insertions(+), 2 deletions(-) create mode 100644 awx/main/notifications/hipchat_backend.py create mode 100644 awx/main/notifications/pagerduty_backend.py diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index a89f460e64..3d4dd9252e 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -13,6 +13,8 @@ from awx.main.models.base import * # noqa from awx.main.notifications.email_backend import CustomEmailBackend from awx.main.notifications.slack_backend import SlackBackend from awx.main.notifications.twilio_backend import TwilioBackend +from awx.main.notifications.pagerduty_backend import PagerDutyBackend +from awx.main.notifications.hipchat_backend import HipChatBackend # Django-JSONField from jsonfield import JSONField @@ -25,7 +27,9 @@ class NotificationTemplate(CommonModel): NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), ('slack', _('Slack'), SlackBackend), - ('twilio', _('Twilio'), TwilioBackend)] + ('twilio', _('Twilio'), TwilioBackend), + ('pagerduty', _('Pagerduty'), PagerDutyBackend), + ('hipchat', _('HipChat'), HipChatBackend)] NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES] CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) @@ -69,6 +73,8 @@ class NotificationTemplate(CommonModel): def send(self, subject, body): recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter) + if not isinstance(recipients, list): + recipients = [recipients] sender = self.notification_configuration.pop(self.notification_class.sender_parameter, None) backend_obj = self.notification_class(**self.notification_configuration) notification_obj = EmailMessage(subject, body, sender, recipients) diff --git a/awx/main/notifications/hipchat_backend.py b/awx/main/notifications/hipchat_backend.py new file mode 100644 index 0000000000..2bb91b3919 --- /dev/null +++ b/awx/main/notifications/hipchat_backend.py @@ -0,0 +1,46 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging + +import requests + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.hipchat_backend') + +class HipChatBackend(BaseEmailBackend): + + init_parameters = {"token": {"label": "Token", "type": "password"}, + "channels": {"label": "Destination Channels", "type": "list"}, + "color": {"label": "Notification Color", "type": "string"}, + "api_url": {"label": "API Url (e.g: https://mycompany.hipchat.com)", "type": "string"}, + "notify": {"label": "Notify channel", "type": "bool"}, + "message_from": {"label": "Label to be shown with notification", "type": "string"}} + recipient_parameter = "channels" + sender_parameter = "message_from" + + def __init__(self, token, color, api_url, notify, fail_silently=False, **kwargs): + super(HipChatBackend, self).__init__(fail_silently=fail_silently) + self.token = token + self.color = color + self.api_url = api_url + self.notify = notify + + def send_messages(self, messages): + sent_messages = 0 + + for m in messages: + for rcp in m.recipients(): + r = requests.post("{}/v2/room/{}/notification".format(self.api_url, rcp), + params={"auth_token": self.token}, + json={"color": self.color, + "message": m.body, + "notify": self.notify, + "from": m.from_email, + "message_format": "text"}) + if r.status_code != 204 and not self.fail_silently: + logger.error("Error sending messages: {}".format(r.text)) + raise Exception("Error sending message to hipchat: {}".format(r.text)) + sent_messages += 1 + return sent_messages diff --git a/awx/main/notifications/pagerduty_backend.py b/awx/main/notifications/pagerduty_backend.py new file mode 100644 index 0000000000..161bb822bc --- /dev/null +++ b/awx/main/notifications/pagerduty_backend.py @@ -0,0 +1,44 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging +import pygerduty + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.pagerduty_backend') + +class PagerDutyBackend(BaseEmailBackend): + + init_parameters = {"subdomain": {"label": "Pagerduty subdomain", "type": "string"}, + "token": {"label": "API Token", "type": "password"}, + "service_key": {"label": "API Service/Integration Key", "type": "string"}, + "client_name": {"label": "Client Identifier", "type": "string"}} + recipient_parameter = "service_key" + sender_parameter = "client_name" + + def __init__(self, subdomain, token, fail_silently=False, **kwargs): + super(PagerDutyBackend, self).__init__(fail_silently=fail_silently) + self.subdomain = subdomain + self.token = token + + def send_messages(self, messages): + sent_messages = 0 + + try: + pager = pygerduty.PagerDuty(self.subdomain, self.token) + except Exception as e: + if not self.fail_silently: + raise + logger.error("Exception connecting to PagerDuty: {}".format(e)) + for m in messages: + try: + pager.trigger_incident(m.recipients()[0], + description=m.subject, + details=m.body, + client=m.from_email) + except Exception as e: + logger.error("Exception sending messages: {}".format(e)) + if not self.fail_silently: + raise + return sent_messages diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index cf2ced368b..d9c4cc43b6 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -41,7 +41,7 @@ class TwilioBackend(BaseEmailBackend): body=m.body) sent_messages += 1 except Exception as e: + logger.error("Exception sending messages: {}".format(e)) if not self.fail_silently: raise - logger.error("Exception sending messages: {}".format(e)) return sent_messages diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 73942d9eec..0f42f235a3 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -82,6 +82,7 @@ psycopg2 pyasn1==0.1.8 pycparser==2.14 pycrypto==2.6.1 +pygerduty==0.32.1 PyJWT==1.4.0 pymongo==2.8 pyOpenSSL==0.15.1 From 7abcb6e30694a2ef9a4c3553856eef0a0e70bbe2 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 10 Feb 2016 17:03:57 -0500 Subject: [PATCH 08/23] Add webhook notification backend --- awx/main/models/notifications.py | 4 ++- awx/main/notifications/hipchat_backend.py | 7 ++--- awx/main/notifications/webhook_backend.py | 32 +++++++++++++++++++++++ 3 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 awx/main/notifications/webhook_backend.py diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 3d4dd9252e..ede9e42795 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -15,6 +15,7 @@ from awx.main.notifications.slack_backend import SlackBackend from awx.main.notifications.twilio_backend import TwilioBackend from awx.main.notifications.pagerduty_backend import PagerDutyBackend from awx.main.notifications.hipchat_backend import HipChatBackend +from awx.main.notifications.webhook_backend import WebhookBackend # Django-JSONField from jsonfield import JSONField @@ -29,7 +30,8 @@ class NotificationTemplate(CommonModel): ('slack', _('Slack'), SlackBackend), ('twilio', _('Twilio'), TwilioBackend), ('pagerduty', _('Pagerduty'), PagerDutyBackend), - ('hipchat', _('HipChat'), HipChatBackend)] + ('hipchat', _('HipChat'), HipChatBackend), + ('webhook', _('Webhook'), WebhookBackend)] NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES] CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) diff --git a/awx/main/notifications/hipchat_backend.py b/awx/main/notifications/hipchat_backend.py index 2bb91b3919..a5b7f561b6 100644 --- a/awx/main/notifications/hipchat_backend.py +++ b/awx/main/notifications/hipchat_backend.py @@ -39,8 +39,9 @@ class HipChatBackend(BaseEmailBackend): "notify": self.notify, "from": m.from_email, "message_format": "text"}) - if r.status_code != 204 and not self.fail_silently: + if r.status_code != 204: logger.error("Error sending messages: {}".format(r.text)) - raise Exception("Error sending message to hipchat: {}".format(r.text)) - sent_messages += 1 + if not self.fail_silently: + raise Exception("Error sending message to hipchat: {}".format(r.text)) + sent_messages += 1 return sent_messages diff --git a/awx/main/notifications/webhook_backend.py b/awx/main/notifications/webhook_backend.py new file mode 100644 index 0000000000..ffef641198 --- /dev/null +++ b/awx/main/notifications/webhook_backend.py @@ -0,0 +1,32 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import logging + +import requests + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.webhook_backend') + +class WebhookBackend(BaseEmailBackend): + + init_parameters = {"url": {"label": "Target URL", "type": "string"}, + "headers": {"label": "HTTP Headers", "type": "object"}} + recipient_parameter = "url" + sender_parameter = None + + def __init__(self, headers, fail_silently=False, **kwargs): + super(WebhookBackend, self).__init__(fail_silently=fail_silently) + + def send_messages(self, messages): + sent_messages = 0 + for m in messages: + r = requests.post("{}".format(m.recipients()[0]), + headers=self.headers) + if r.status_code >= 400: + logger.error("Error sending notification webhook: {}".format(r.text)) + if not self.fail_silently: + raise Exception("Error sending notification webhook: {}".format(r.text)) + sent_messages += 1 + return sent_messages From 9d6739045ab23e7a966faea3caf1e653f273d70b Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Thu, 11 Feb 2016 15:34:27 -0500 Subject: [PATCH 09/23] Implement irc notification backend --- awx/main/models/notifications.py | 4 +- awx/main/notifications/irc_backend.py | 93 +++++++++++++++++++++++++ awx/main/notifications/slack_backend.py | 2 +- requirements/requirements.txt | 1 + 4 files changed, 98 insertions(+), 2 deletions(-) create mode 100644 awx/main/notifications/irc_backend.py diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index ede9e42795..4fc005256b 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -16,6 +16,7 @@ from awx.main.notifications.twilio_backend import TwilioBackend from awx.main.notifications.pagerduty_backend import PagerDutyBackend from awx.main.notifications.hipchat_backend import HipChatBackend from awx.main.notifications.webhook_backend import WebhookBackend +from awx.main.notifications.irc_backend import IrcBackend # Django-JSONField from jsonfield import JSONField @@ -31,7 +32,8 @@ class NotificationTemplate(CommonModel): ('twilio', _('Twilio'), TwilioBackend), ('pagerduty', _('Pagerduty'), PagerDutyBackend), ('hipchat', _('HipChat'), HipChatBackend), - ('webhook', _('Webhook'), WebhookBackend)] + ('webhook', _('Webhook'), WebhookBackend), + ('irc', _('IRC'), IrcBackend)] NOTIFICATION_TYPE_CHOICES = [(x[0], x[1]) for x in NOTIFICATION_TYPES] CLASS_FOR_NOTIFICATION_TYPE = dict([(x[0], x[2]) for x in NOTIFICATION_TYPES]) diff --git a/awx/main/notifications/irc_backend.py b/awx/main/notifications/irc_backend.py new file mode 100644 index 0000000000..2b0944b74a --- /dev/null +++ b/awx/main/notifications/irc_backend.py @@ -0,0 +1,93 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import time +import ssl +import logging + +import irc.client + +from django.core.mail.backends.base import BaseEmailBackend + +logger = logging.getLogger('awx.main.notifications.irc_backend') + +class IrcBackend(BaseEmailBackend): + + init_parameters = {"server": {"label": "IRC Server Address", "type": "string"}, + "port": {"label": "IRC Server Port", "type": "int"}, + "nickname": {"label": "IRC Nick", "type": "string"}, + "password": {"label": "IRC Server Password", "type": "password"}, + "use_ssl": {"label": "SSL Connection", "type": "bool"}, + "targets": {"label": "Destination Channels or Users", "type": "list"}} + recipient_parameter = "targets" + sender_parameter = None + + def __init__(self, server, port, nickname, password, use_ssl, fail_silently=False, **kwargs): + super(IrcBackend, self).__init__(fail_silently=fail_silently) + self.server = server + self.port = port + self.nickname = nickname + self.password = password if password != "" else None + self.use_ssl = use_ssl + self.connection = None + + def open(self): + if self.connection is not None: + return False + if self.use_ssl: + connection_factory = irc.connection.Factory(wrapper=ssl.wrap_socket) + else: + connection_factory = irc.connection.Factory() + try: + self.reactor = irc.client.Reactor() + self.connection = self.reactor.server().connect( + self.server, + self.port, + self.nickname, + password=self.password, + connect_factory=connection_factory, + ) + except irc.client.ServerConnectionError as e: + logger.error("Exception connecting to irc server: {}".format(e)) + if not self.fail_silently: + raise + return True + + def close(self): + if self.connection is None: + return + self.connection = None + + def on_connect(self, connection, event): + for c in self.channels: + if irc.client.is_channel(c): + connection.join(c) + else: + for m in self.channels[c]: + connection.privmsg(c, m.subject) + self.channels_sent += 1 + + def on_join(self, connection, event): + for m in self.channels[event.target]: + connection.privmsg(event.target, m.subject) + self.channels_sent += 1 + + def send_messages(self, messages): + if self.connection is None: + self.open() + self.channels = {} + self.channels_sent = 0 + for m in messages: + for r in m.recipients(): + if r not in self.channels: + self.channels[r] = [] + self.channels[r].append(m) + self.connection.add_global_handler("welcome", self.on_connect) + self.connection.add_global_handler("join", self.on_join) + start_time = time.time() + process_time = time.time() + while self.channels_sent < len(self.channels) and (process_time-start_time) < 60: + self.reactor.process_once(0.1) + process_time = time.time() + self.reactor.disconnect_all() + return self.channels_sent diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 950d5c2c6e..3bf4f32114 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -44,7 +44,7 @@ class SlackBackend(BaseEmailBackend): self.connection.rtm_send_message(r, m.body) sent_messages += 1 except Exception as e: + logger.error("Exception sending messages: {}".format(e)) if not self.fail_silently: raise - logger.error("Exception sending messages: {}".format(e)) return sent_messages diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 0f42f235a3..e8c62488f5 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -47,6 +47,7 @@ httplib2==0.9 idna==2.0 importlib==1.0.3 ipaddress==1.0.14 +irc==13.3.1 iso8601==0.1.10 isodate==0.5.1 jsonpatch==1.11 From dde70dafec20097bfdef7263c5ce2cc592195231 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 17 Feb 2016 15:18:18 +0000 Subject: [PATCH 10/23] Refactor NotificationTemplate to Notifier --- awx/api/metadata.py | 4 +- awx/api/serializers.py | 40 +++---- awx/api/urls.py | 36 +++--- awx/api/views.py | 128 +++++++++++----------- awx/main/access.py | 8 +- awx/main/models/__init__.py | 2 +- awx/main/models/activity_stream.py | 2 +- awx/main/models/base.py | 18 +-- awx/main/models/inventory.py | 10 +- awx/main/models/jobs.py | 10 +- awx/main/models/notifications.py | 12 +- awx/main/models/projects.py | 10 +- awx/main/models/unified_jobs.py | 2 +- awx/main/notifications/webhook_backend.py | 2 + awx/main/signals.py | 2 +- 15 files changed, 145 insertions(+), 141 deletions(-) diff --git a/awx/api/metadata.py b/awx/api/metadata.py index b5e6d7043a..05bd11a4c7 100644 --- a/awx/api/metadata.py +++ b/awx/api/metadata.py @@ -12,7 +12,7 @@ from rest_framework import serializers from rest_framework.request import clone_request # Ansible Tower -from awx.main.models import InventorySource, NotificationTemplate +from awx.main.models import InventorySource, Notifier class Metadata(metadata.SimpleMetadata): @@ -58,7 +58,7 @@ class Metadata(metadata.SimpleMetadata): # Special handling of notification configuration where the required properties # are conditional on the type selected. if field.field_name == 'notification_configuration': - for (notification_type_name, notification_tr_name, notification_type_class) in NotificationTemplate.NOTIFICATION_TYPES: + for (notification_type_name, notification_tr_name, notification_type_class) in Notifier.NOTIFICATION_TYPES: field_info[notification_type_name] = notification_type_class.init_parameters # Update type of fields returned... diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 7bc25f532d..3a324ce21f 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -769,9 +769,9 @@ class OrganizationSerializer(BaseSerializer): teams = reverse('api:organization_teams_list', args=(obj.pk,)), activity_stream = reverse('api:organization_activity_stream_list', args=(obj.pk,)), notifiers = reverse('api:organization_notifiers_list', args=(obj.pk,)), - notifiers_any = reverse('api:organization_notifications_any_list', args=(obj.pk,)), - notifiers_success = reverse('api:organization_notifications_success_list', args=(obj.pk,)), - notifiers_error = reverse('api:organization_notifications_error_list', args=(obj.pk,)), + notifiers_any = reverse('api:organization_notifiers_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:organization_notifiers_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:organization_notifiers_error_list', args=(obj.pk,)), )) return res @@ -851,9 +851,9 @@ class ProjectSerializer(UnifiedJobTemplateSerializer, ProjectOptionsSerializer): project_updates = reverse('api:project_updates_list', args=(obj.pk,)), schedules = reverse('api:project_schedules_list', args=(obj.pk,)), activity_stream = reverse('api:project_activity_stream_list', args=(obj.pk,)), - notifiers_any = reverse('api:project_notifications_any_list', args=(obj.pk,)), - notifiers_success = reverse('api:project_notifications_success_list', args=(obj.pk,)), - notifiers_error = reverse('api:project_notifications_error_list', args=(obj.pk,)), + notifiers_any = reverse('api:project_notifiers_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:project_notifiers_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:project_notifiers_error_list', args=(obj.pk,)), )) # Backwards compatibility. if obj.current_update: @@ -1298,9 +1298,9 @@ class InventorySourceSerializer(UnifiedJobTemplateSerializer, InventorySourceOpt activity_stream = reverse('api:inventory_activity_stream_list', args=(obj.pk,)), hosts = reverse('api:inventory_source_hosts_list', args=(obj.pk,)), groups = reverse('api:inventory_source_groups_list', args=(obj.pk,)), - notifiers_any = reverse('api:inventory_source_notifications_any_list', args=(obj.pk,)), - notifiers_success = reverse('api:inventory_source_notifications_success_list', args=(obj.pk,)), - notifiers_error = reverse('api:inventory_source_notifications_error_list', args=(obj.pk,)), + notifiers_any = reverse('api:inventory_source_notifiers_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:inventory_source_notifiers_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:inventory_source_notifiers_error_list', args=(obj.pk,)), )) if obj.inventory and obj.inventory.active: res['inventory'] = reverse('api:inventory_detail', args=(obj.inventory.pk,)) @@ -1564,9 +1564,9 @@ class JobTemplateSerializer(UnifiedJobTemplateSerializer, JobOptionsSerializer): schedules = reverse('api:job_template_schedules_list', args=(obj.pk,)), activity_stream = reverse('api:job_template_activity_stream_list', args=(obj.pk,)), launch = reverse('api:job_template_launch', args=(obj.pk,)), - notifiers_any = reverse('api:job_template_notifications_any_list', args=(obj.pk,)), - notifiers_success = reverse('api:job_template_notifications_success_list', args=(obj.pk,)), - notifiers_error = reverse('api:job_template_notifications_error_list', args=(obj.pk,)), + notifiers_any = reverse('api:job_template_notifiers_any_list', args=(obj.pk,)), + notifiers_success = reverse('api:job_template_notifiers_success_list', args=(obj.pk,)), + notifiers_error = reverse('api:job_template_notifiers_error_list', args=(obj.pk,)), )) if obj.host_config_key: res['callback'] = reverse('api:job_template_callback', args=(obj.pk,)) @@ -2053,22 +2053,24 @@ class JobLaunchSerializer(BaseSerializer): attrs = super(JobLaunchSerializer, self).validate(attrs) return attrs -class NotificationTemplateSerializer(BaseSerializer): +class NotifierSerializer(BaseSerializer): class Meta: - model = NotificationTemplate + model = Notifier fields = ('*', 'organization', 'notification_type', 'notification_configuration') def get_related(self, obj): - res = super(NotificationTemplateSerializer, self).get_related(obj) + res = super(NotifierSerializer, self).get_related(obj) res.update(dict( - test = reverse('api:notification_template_test', args=(obj.pk,)), - notifications = reverse('api:notification_template_notification_list', args=(obj.pk,)), + test = reverse('api:notifier_test', args=(obj.pk,)), + notifications = reverse('api:notifier_notification_list', args=(obj.pk,)), )) + if obj.organization and obj.organization.active: + res['organization'] = reverse('api:organization_detail', args=(obj.organization.pk,)) return res def validate(self, attrs): - notification_class = NotificationTemplate.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] + notification_class = Notifier.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] missing_fields = [] for field in notification_class.init_parameters: if field not in attrs['notification_configuration']: @@ -2088,7 +2090,7 @@ class NotificationSerializer(BaseSerializer): def get_related(self, obj): res = super(NotificationSerializer, self).get_related(obj) res.update(dict( - notification_template = reverse('api:notification_template_detail', args=(obj.notifier.pk,)), + notifier = reverse('api:notifier_detail', args=(obj.notifier.pk,)), )) return res diff --git a/awx/api/urls.py b/awx/api/urls.py index 7e55e46a5c..d9e0fca5cb 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -21,9 +21,9 @@ organization_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/teams/$', 'organization_teams_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'organization_activity_stream_list'), url(r'^(?P[0-9]+)/notifiers/$', 'organization_notifiers_list'), - url(r'^(?P[0-9]+)/notifications_any/$', 'organization_notifications_any_list'), - url(r'^(?P[0-9]+)/notifications_error/$', 'organization_notifications_error_list'), - url(r'^(?P[0-9]+)/notifications_success/$', 'organization_notifications_success_list'), + url(r'^(?P[0-9]+)/notifiers_any/$', 'organization_notifiers_any_list'), + url(r'^(?P[0-9]+)/notifiers_error/$', 'organization_notifiers_error_list'), + url(r'^(?P[0-9]+)/notifiers_success/$', 'organization_notifiers_success_list'), ) user_urls = patterns('awx.api.views', @@ -48,9 +48,9 @@ project_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/project_updates/$', 'project_updates_list'), url(r'^(?P[0-9]+)/activity_stream/$', 'project_activity_stream_list'), url(r'^(?P[0-9]+)/schedules/$', 'project_schedules_list'), - url(r'^(?P[0-9]+)/notifications_any/$', 'project_notifications_any_list'), - url(r'^(?P[0-9]+)/notifications_error/$', 'project_notifications_error_list'), - url(r'^(?P[0-9]+)/notifications_success/$', 'project_notifications_success_list'), + url(r'^(?P[0-9]+)/notifiers_any/$', 'project_notifiers_any_list'), + url(r'^(?P[0-9]+)/notifiers_error/$', 'project_notifiers_error_list'), + url(r'^(?P[0-9]+)/notifiers_success/$', 'project_notifiers_success_list'), ) project_update_urls = patterns('awx.api.views', @@ -128,9 +128,9 @@ inventory_source_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/schedules/$', 'inventory_source_schedules_list'), url(r'^(?P[0-9]+)/groups/$', 'inventory_source_groups_list'), url(r'^(?P[0-9]+)/hosts/$', 'inventory_source_hosts_list'), - url(r'^(?P[0-9]+)/notifications_any/$', 'inventory_source_notifications_any_list'), - url(r'^(?P[0-9]+)/notifications_error/$', 'inventory_source_notifications_error_list'), - url(r'^(?P[0-9]+)/notifications_success/$', 'inventory_source_notifications_success_list'), + url(r'^(?P[0-9]+)/notifiers_any/$', 'inventory_source_notifiers_any_list'), + url(r'^(?P[0-9]+)/notifiers_error/$', 'inventory_source_notifiers_error_list'), + url(r'^(?P[0-9]+)/notifiers_success/$', 'inventory_source_notifiers_success_list'), ) inventory_update_urls = patterns('awx.api.views', @@ -165,9 +165,9 @@ job_template_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/schedules/$', 'job_template_schedules_list'), url(r'^(?P[0-9]+)/survey_spec/$', 'job_template_survey_spec'), url(r'^(?P[0-9]+)/activity_stream/$', 'job_template_activity_stream_list'), - url(r'^(?P[0-9]+)/notifications_any/$', 'job_template_notifications_any_list'), - url(r'^(?P[0-9]+)/notifications_error/$', 'job_template_notifications_error_list'), - url(r'^(?P[0-9]+)/notifications_success/$', 'job_template_notifications_success_list'), + url(r'^(?P[0-9]+)/notifiers_any/$', 'job_template_notifiers_any_list'), + url(r'^(?P[0-9]+)/notifiers_error/$', 'job_template_notifiers_error_list'), + url(r'^(?P[0-9]+)/notifiers_success/$', 'job_template_notifiers_success_list'), ) job_urls = patterns('awx.api.views', @@ -225,11 +225,11 @@ system_job_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/cancel/$', 'system_job_cancel'), ) -notification_template_urls = patterns('awx.api.views', - url(r'^$', 'notification_template_list'), - url(r'^(?P[0-9]+)/$', 'notification_template_detail'), - url(r'^(?P[0-9]+)/test/$', 'notification_template_test'), - url(r'^(?P[0-9]+)/notifications/$', 'notification_template_notification_list'), +notifier_urls = patterns('awx.api.views', + url(r'^$', 'notifier_list'), + url(r'^(?P[0-9]+)/$', 'notifier_detail'), + url(r'^(?P[0-9]+)/test/$', 'notifier_test'), + url(r'^(?P[0-9]+)/notifications/$', 'notifier_notification_list'), ) notification_urls = patterns('awx.api.views', @@ -285,7 +285,7 @@ v1_urls = patterns('awx.api.views', url(r'^ad_hoc_command_events/', include(ad_hoc_command_event_urls)), url(r'^system_job_templates/', include(system_job_template_urls)), url(r'^system_jobs/', include(system_job_urls)), - url(r'^notification_templates/', include(notification_template_urls)), + url(r'^notifiers/', include(notifier_urls)), url(r'^notifications/', include(notification_urls)), url(r'^unified_job_templates/$', 'unified_job_template_list'), url(r'^unified_jobs/$', 'unified_job_list'), diff --git a/awx/api/views.py b/awx/api/views.py index be1bd3b609..439de3f845 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -135,7 +135,7 @@ class ApiV1RootView(APIView): data['system_job_templates'] = reverse('api:system_job_template_list') data['system_jobs'] = reverse('api:system_job_list') data['schedules'] = reverse('api:schedule_list') - data['notification_templates'] = reverse('api:notification_template_list') + data['notifiers'] = reverse('api:notifier_list') data['notifications'] = reverse('api:notification_list') data['unified_job_templates'] = reverse('api:unified_job_template_list') data['unified_jobs'] = reverse('api:unified_job_list') @@ -687,32 +687,32 @@ class OrganizationActivityStreamList(SubListAPIView): class OrganizationNotifiersList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Organization - relationship = 'notification_templates' + relationship = 'notifiers' parent_key = 'organization' -class OrganizationNotificationsAnyList(SubListCreateAttachDetachAPIView): +class OrganizationNotifiersAnyList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Organization - relationship = 'notification_any' + relationship = 'notifiers_any' -class OrganizationNotificationsErrorList(SubListCreateAttachDetachAPIView): +class OrganizationNotifiersErrorList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Organization - relationship = 'notification_erros' + relationship = 'notifiers_error' -class OrganizationNotificationsSuccessList(SubListCreateAttachDetachAPIView): +class OrganizationNotifiersSuccessList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Organization - relationship = 'notification_success' + relationship = 'notifiers_success' class TeamList(ListCreateAPIView): @@ -879,26 +879,26 @@ class ProjectActivityStreamList(SubListAPIView): return qs.filter(project=parent) return qs.filter(Q(project=parent) | Q(credential__in=parent.credential)) -class ProjectNotificationsAnyList(SubListCreateAttachDetachAPIView): +class ProjectNotifiersAnyList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Project - relationship = 'notification_any' + relationship = 'notifiers_any' -class ProjectNotificationsErrorList(SubListCreateAttachDetachAPIView): +class ProjectNotifiersErrorList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Project - relationship = 'notification_errors' + relationship = 'notifiers_error' -class ProjectNotificationsSuccessList(SubListCreateAttachDetachAPIView): +class ProjectNotifiersSuccessList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = Project - relationship = 'notification_success' + relationship = 'notifiers_success' class ProjectUpdatesList(SubListAPIView): @@ -1781,26 +1781,26 @@ class InventorySourceActivityStreamList(SubListAPIView): # Okay, let it through. return super(type(self), self).get(request, *args, **kwargs) -class InventorySourceNotificationsAnyList(SubListCreateAttachDetachAPIView): +class InventorySourceNotifiersAnyList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = InventorySource - relationship = 'notification_any' + relationship = 'notifiers_any' -class InventorySourceNotificationsErrorList(SubListCreateAttachDetachAPIView): +class InventorySourceNotifiersErrorList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = InventorySource - relationship = 'notification_errors' + relationship = 'notifiers_error' -class InventorySourceNotificationsSuccessList(SubListCreateAttachDetachAPIView): +class InventorySourceNotifiersSuccessList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = InventorySource - relationship = 'notification_success' + relationship = 'notifiers_success' class InventorySourceHostsList(SubListAPIView): @@ -2027,26 +2027,26 @@ class JobTemplateActivityStreamList(SubListAPIView): # Okay, let it through. return super(type(self), self).get(request, *args, **kwargs) -class JobTemplateNotificationsAnyList(SubListCreateAttachDetachAPIView): +class JobTemplateNotifiersAnyList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = JobTemplate - relationship = 'notification_any' + relationship = 'notifiers_any' -class JobTemplateNotificationsErrorList(SubListCreateAttachDetachAPIView): +class JobTemplateNotifiersErrorList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = JobTemplate - relationship = 'notification_errors' + relationship = 'notifiers_error' -class JobTemplateNotificationsSuccessList(SubListCreateAttachDetachAPIView): +class JobTemplateNotifiersSuccessList(SubListCreateAttachDetachAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer parent_model = JobTemplate - relationship = 'notification_success' + relationship = 'notifiers_success' class JobTemplateCallback(GenericAPIView): @@ -3032,22 +3032,22 @@ class AdHocCommandStdout(UnifiedJobStdout): model = AdHocCommand new_in_220 = True -class NotificationTemplateList(ListCreateAPIView): +class NotifierList(ListCreateAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer new_in_300 = True -class NotificationTemplateDetail(RetrieveUpdateDestroyAPIView): +class NotifierDetail(RetrieveUpdateDestroyAPIView): - model = NotificationTemplate - serializer_class = NotificationTemplateSerializer + model = Notifier + serializer_class = NotifierSerializer new_in_300 = True -class NotificationTemplateTest(GenericAPIView): +class NotifierTest(GenericAPIView): - view_name = 'Notification Template Test' - model = NotificationTemplate + view_name = 'Notifier Test' + model = Notifier serializer_class = EmptySerializer new_in_300 = True @@ -3063,11 +3063,11 @@ class NotificationTemplateTest(GenericAPIView): headers=headers, status=status.HTTP_202_ACCEPTED) -class NotificationTemplateNotificationList(SubListAPIView): +class NotifierNotificationList(SubListAPIView): model = Notification serializer_class = NotificationSerializer - parent_model = NotificationTemplate + parent_model = Notifier relationship = 'notifications' parent_key = 'notifier' @@ -3079,7 +3079,7 @@ class NotificationList(ListAPIView): class NotificationDetail(RetrieveAPIView): - model = NotificationTemplate + model = Notification serializer_class = NotificationSerializer new_in_300 = True diff --git a/awx/main/access.py b/awx/main/access.py index 3ffbaf7f85..fb89dd30d8 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -1484,11 +1484,11 @@ class ScheduleAccess(BaseAccess): else: return False -class NotificationTemplateAccess(BaseAccess): +class NotifierAccess(BaseAccess): ''' - I can see/use a notification template if I have permission to + I can see/use a notifier if I have permission to ''' - model = NotificationTemplate + model = Notifier def get_queryset(self): qs = self.model.objects.filter(active=True).distinct() @@ -1708,5 +1708,5 @@ register_access(UnifiedJob, UnifiedJobAccess) register_access(ActivityStream, ActivityStreamAccess) register_access(CustomInventoryScript, CustomInventoryScriptAccess) register_access(TowerSettings, TowerSettingsAccess) -register_access(NotificationTemplate, NotificationTemplateAccess) +register_access(Notifier, NotifierAccess) register_access(Notification, NotificationAccess) diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 2397b6137b..cf8235ba18 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -61,5 +61,5 @@ activity_stream_registrar.connect(AdHocCommand) activity_stream_registrar.connect(Schedule) activity_stream_registrar.connect(CustomInventoryScript) activity_stream_registrar.connect(TowerSettings) -activity_stream_registrar.connect(NotificationTemplate) +activity_stream_registrar.connect(Notifier) activity_stream_registrar.connect(Notification) diff --git a/awx/main/models/activity_stream.py b/awx/main/models/activity_stream.py index 12a54c7af2..dfada31484 100644 --- a/awx/main/models/activity_stream.py +++ b/awx/main/models/activity_stream.py @@ -53,7 +53,7 @@ class ActivityStream(models.Model): ad_hoc_command = models.ManyToManyField("AdHocCommand", blank=True) schedule = models.ManyToManyField("Schedule", blank=True) custom_inventory_script = models.ManyToManyField("CustomInventoryScript", blank=True) - notification_template = models.ManyToManyField("NotificationTemplate", blank=True) + notifier = models.ManyToManyField("Notifier", blank=True) notification = models.ManyToManyField("Notification", blank=True) def get_absolute_url(self): diff --git a/awx/main/models/base.py b/awx/main/models/base.py index f3f158855c..c4edfbd8ba 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -343,20 +343,20 @@ class NotificationFieldsModel(BaseModel): class Meta: abstract = True - notification_errors = models.ManyToManyField( - "NotificationTemplate", + notifiers_error = models.ManyToManyField( + "Notifier", blank=True, - related_name='%(class)s_notifications_for_errors' + related_name='%(class)s_notifiers_for_errors' ) - notification_success = models.ManyToManyField( - "NotificationTemplate", + notifiers_success = models.ManyToManyField( + "Notifier", blank=True, - related_name='%(class)s_notifications_for_success' + related_name='%(class)s_notifiers_for_success' ) - notification_any = models.ManyToManyField( - "NotificationTemplate", + notifiers_any = models.ManyToManyField( + "Notifier", blank=True, - related_name='%(class)s_notifications_for_any' + related_name='%(class)s_notifiers_for_any' ) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index febf010f20..b0acf0a90b 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -23,7 +23,7 @@ from awx.main.managers import HostManager from awx.main.models.base import * # noqa from awx.main.models.jobs import Job from awx.main.models.unified_jobs import * # noqa -from awx.main.models.notifications import NotificationTemplate +from awx.main.models.notifications import Notifier from awx.main.utils import ignore_inventory_computed_fields, _inventory_updates __all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript'] @@ -1184,10 +1184,10 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): @property def notifiers(self): # Return all notifiers defined on the Project, and on the Organization for each trigger type - base_notifiers = NotificationTemplate.objects.filter(active=True) - error_notifiers = list(base_notifiers.filter(organization_notifications_for_errors__in=self)) - success_notifiers = list(base_notifiers.filter(organization_notifications_for_success__in=self)) - any_notifiers = list(base_notifiers.filter(organization_notifications_for_any__in=self)) + base_notifiers = Notifier.objects.filter(active=True) + error_notifiers = list(base_notifiers.filter(organization_notifiers_for_errors__in=self)) + success_notifiers = list(base_notifiers.filter(organization_notifiers_for_success__in=self)) + any_notifiers = list(base_notifiers.filter(organization_notifiers_for_any__in=self)) return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) def clean_source(self): diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 42f7ccf676..dd772d695d 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -22,7 +22,7 @@ from jsonfield import JSONField from awx.main.constants import CLOUD_PROVIDERS from awx.main.models.base import * # noqa from awx.main.models.unified_jobs import * # noqa -from awx.main.models.notifications import NotificationTemplate +from awx.main.models.notifications import Notifier from awx.main.utils import decrypt_field, ignore_inventory_computed_fields from awx.main.utils import emit_websocket_notification from awx.main.redact import PlainTextCleaner @@ -336,10 +336,10 @@ class JobTemplate(UnifiedJobTemplate, JobOptions): # Return all notifiers defined on the Job Template, on the Project, and on the Organization for each trigger type # TODO: Currently there is no org fk on project so this will need to be added once that is # available after the rbac pr - base_notifiers = NotificationTemplate.objects.filter(active=True) - error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_errors__in=[self, self.project])) - success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_success__in=[self, self.project])) - any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_any__in=[self, self.project])) + base_notifiers = Notifier.objects.filter(active=True) + error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_errors__in=[self, self.project])) + success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_success__in=[self, self.project])) + any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_any__in=[self, self.project])) return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) class Job(UnifiedJob, JobOptions): diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 4fc005256b..2e1bb9df2f 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -23,9 +23,9 @@ from jsonfield import JSONField logger = logging.getLogger('awx.main.models.notifications') -__all__ = ['NotificationTemplate', 'Notification'] +__all__ = ['Notifier', 'Notification'] -class NotificationTemplate(CommonModel): +class Notifier(CommonModel): NOTIFICATION_TYPES = [('email', _('Email'), CustomEmailBackend), ('slack', _('Slack'), SlackBackend), @@ -45,7 +45,7 @@ class NotificationTemplate(CommonModel): blank=False, null=True, on_delete=models.SET_NULL, - related_name='notification_templates', + related_name='notifiers', ) notification_type = models.CharField( @@ -56,7 +56,7 @@ class NotificationTemplate(CommonModel): notification_configuration = JSONField(blank=False) def get_absolute_url(self): - return reverse('api:notification_template_detail', args=(self.pk,)) + return reverse('api:notifier_detail', args=(self.pk,)) @property def notification_class(self): @@ -100,7 +100,7 @@ class Notification(CreatedModifiedModel): ordering = ('pk',) notifier = models.ForeignKey( - 'NotificationTemplate', + 'Notifier', related_name='notifications', on_delete=models.CASCADE, editable=False @@ -122,7 +122,7 @@ class Notification(CreatedModifiedModel): ) notification_type = models.CharField( max_length = 32, - choices=NotificationTemplate.NOTIFICATION_TYPE_CHOICES, + choices=Notifier.NOTIFICATION_TYPE_CHOICES, ) recipients = models.TextField( blank=True, diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 730604d3e4..8a320e3cfc 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -21,7 +21,7 @@ from django.utils.timezone import now, make_aware, get_default_timezone from awx.lib.compat import slugify from awx.main.models.base import * # noqa from awx.main.models.jobs import Job -from awx.main.models.notifications import NotificationTemplate +from awx.main.models.notifications import Notifier from awx.main.models.unified_jobs import * # noqa from awx.main.utils import update_scm_url @@ -316,16 +316,16 @@ class Project(UnifiedJobTemplate, ProjectOptions): # Return all notifiers defined on the Project, and on the Organization for each trigger type # TODO: Currently there is no org fk on project so this will need to be added back once that is # available after the rbac pr - base_notifiers = NotificationTemplate.objects.filter(active=True) + base_notifiers = Notifier.objects.filter(active=True) # error_notifiers = list(base_notifiers.filter(Q(project_notifications_for_errors__in=self) | # Q(organization_notifications_for_errors__in=self.organization))) # success_notifiers = list(base_notifiers.filter(Q(project_notifications_for_success__in=self) | # Q(organization_notifications_for_success__in=self.organization))) # any_notifiers = list(base_notifiers.filter(Q(project_notifications_for_any__in=self) | # Q(organization_notifications_for_any__in=self.organization))) - error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_errors=self)) - success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_success=self)) - any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifications_for_any=self)) + error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_errors=self)) + success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_success=self)) + any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_any=self)) return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) def get_absolute_url(self): diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index c6ea2b082b..986be923fb 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -304,7 +304,7 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio Return notifiers relevant to this Unified Job Template ''' # NOTE: Derived classes should implement - return NotificationTemplate.objects.none() + return Notifier.objects.none() def create_unified_job(self, **kwargs): ''' diff --git a/awx/main/notifications/webhook_backend.py b/awx/main/notifications/webhook_backend.py index ffef641198..5bdbff0e02 100644 --- a/awx/main/notifications/webhook_backend.py +++ b/awx/main/notifications/webhook_backend.py @@ -17,12 +17,14 @@ class WebhookBackend(BaseEmailBackend): sender_parameter = None def __init__(self, headers, fail_silently=False, **kwargs): + self.headers = headers super(WebhookBackend, self).__init__(fail_silently=fail_silently) def send_messages(self, messages): sent_messages = 0 for m in messages: r = requests.post("{}".format(m.recipients()[0]), + data=m.body, headers=self.headers) if r.status_code >= 400: logger.error("Error sending notification webhook: {}".format(r.text)) diff --git a/awx/main/signals.py b/awx/main/signals.py index f4d0014905..29c5c7d016 100644 --- a/awx/main/signals.py +++ b/awx/main/signals.py @@ -307,7 +307,7 @@ model_serializer_mapping = { Job: JobSerializer, AdHocCommand: AdHocCommandSerializer, TowerSettings: TowerSettingsSerializer, - NotificationTemplate: NotificationTemplateSerializer, + Notifier: NotifierSerializer, Notification: NotificationSerializer, } From 52974648dfc8783b095fd14861360f5b2859628f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 17 Feb 2016 16:01:54 +0000 Subject: [PATCH 11/23] Notification configuration type checking --- awx/api/serializers.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 3a324ce21f..d901184ee0 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2059,6 +2059,9 @@ class NotifierSerializer(BaseSerializer): model = Notifier fields = ('*', 'organization', 'notification_type', 'notification_configuration') + type_map = {"string": str, "int": int, "bool": bool, "list": list, + "password": str, "object": dict} + def get_related(self, obj): res = super(NotifierSerializer, self).get_related(obj) res.update(dict( @@ -2072,12 +2075,22 @@ class NotifierSerializer(BaseSerializer): def validate(self, attrs): notification_class = Notifier.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] missing_fields = [] + incorrect_type_fields = [] for field in notification_class.init_parameters: if field not in attrs['notification_configuration']: missing_fields.append(field) - # TODO: Type checks + continue + field_val = attrs['notification_configuration'][field] + field_type = notification_class.init_parameters[field]['type'] + expected_type = self.type_map[field_type] + if not isinstance(field_val, expected_type): + incorrect_type_fields.append((field, field_type)) if missing_fields: - raise serializers.ValidationError("Missing required fields for Notification Configuration: {}".format(missing_fields)) + error_list = ["Missing required fields for Notification Configuration: {}".format(missing_fields)] + for type_field_error in incorrect_type_fields: + error_list.append("Configuration field {} incorrect type, expected {}".format(type_field_error[0], + type_field_error[1])) + raise serializers.ValidationError(error_list) return attrs class NotificationSerializer(BaseSerializer): From eb3d663d18f904aa263278aa99298435d6078767 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Sun, 21 Feb 2016 23:11:17 -0500 Subject: [PATCH 12/23] Support notification password field encryption Modify encrypt_field and decrypt_field to support sub-fields under a dictionary object. It still uses the parent key when encrypting. --- awx/api/serializers.py | 32 ++++++++++++++++++++++++++------ awx/main/models/notifications.py | 31 +++++++++++++++++++++++++++++++ awx/main/utils.py | 9 ++++++--- 3 files changed, 63 insertions(+), 9 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d901184ee0..04f5e241c3 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2059,8 +2059,20 @@ class NotifierSerializer(BaseSerializer): model = Notifier fields = ('*', 'organization', 'notification_type', 'notification_configuration') - type_map = {"string": str, "int": int, "bool": bool, "list": list, - "password": str, "object": dict} + type_map = {"string": (str, unicode), + "int": (int,), + "bool": (bool,), + "list": (list,), + "password": (str, unicode), + "object": (dict,)} + + def to_representation(self, obj): + ret = super(NotifierSerializer, self).to_representation(obj) + for field in obj.notification_class.init_parameters: + if field in ret['notification_configuration'] and \ + force_text(ret['notification_configuration'][field]).startswith('$encrypted$'): + ret['notification_configuration'][field] = '$encrypted$' + return ret def get_related(self, obj): res = super(NotifierSerializer, self).get_related(obj) @@ -2076,20 +2088,28 @@ class NotifierSerializer(BaseSerializer): notification_class = Notifier.CLASS_FOR_NOTIFICATION_TYPE[attrs['notification_type']] missing_fields = [] incorrect_type_fields = [] + if 'notification_configuration' not in attrs: + return attrs for field in notification_class.init_parameters: if field not in attrs['notification_configuration']: missing_fields.append(field) continue field_val = attrs['notification_configuration'][field] field_type = notification_class.init_parameters[field]['type'] - expected_type = self.type_map[field_type] - if not isinstance(field_val, expected_type): + expected_types = self.type_map[field_type] + if not type(field_val) in expected_types: incorrect_type_fields.append((field, field_type)) + continue + if field_type == "password" and field_val.startswith('$encrypted$'): + missing_fields.append(field) + error_list = [] if missing_fields: - error_list = ["Missing required fields for Notification Configuration: {}".format(missing_fields)] + error_list.append("Missing required fields for Notification Configuration: {}".format(missing_fields)) + if incorrect_type_fields: for type_field_error in incorrect_type_fields: - error_list.append("Configuration field {} incorrect type, expected {}".format(type_field_error[0], + error_list.append("Configuration field '{}' incorrect type, expected {}".format(type_field_error[0], type_field_error[1])) + if error_list: raise serializers.ValidationError(error_list) return attrs diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 2e1bb9df2f..d6fc9d31b8 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -10,6 +10,7 @@ from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_str from awx.main.models.base import * # noqa +from awx.main.utils import encrypt_field, decrypt_field from awx.main.notifications.email_backend import CustomEmailBackend from awx.main.notifications.slack_backend import SlackBackend from awx.main.notifications.twilio_backend import TwilioBackend @@ -62,6 +63,31 @@ class Notifier(CommonModel): def notification_class(self): return self.CLASS_FOR_NOTIFICATION_TYPE[self.notification_type] + def save(self, *args, **kwargs): + new_instance = not bool(self.pk) + update_fields = kwargs.get('update_fields', []) + for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", + self.notification_class.init_parameters): + if new_instance: + value = getattr(self.notification_configuration, field, '') + setattr(self, '_saved_{}'.format(field), value) + self.notification_configuration[field] = '' + else: + encrypted = encrypt_field(self, 'notification_configuration', subfield=field) + self.notification_configuration[field] = encrypted + if 'notification_configuration' not in update_fields: + update_fields.append('notification_configuration') + super(Notifier, self).save(*args, **kwargs) + if new_instance: + update_fields = [] + for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", + self.notification_class.init_parameters): + saved_value = getattr(self, '_saved_{}'.format(field), '') + setattr(self.notification_configuration, field, saved_value) + if 'notification_configuration' not in update_fields: + update_fields.append('notification_configuration') + self.save(update_fields=update_fields) + @property def recipients(self): return self.notification_configuration[self.notification_class.recipient_parameter] @@ -76,6 +102,11 @@ class Notifier(CommonModel): return notification def send(self, subject, body): + for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", + self.notification_class.init_parameters): + self.notification_configuration[field] = decrypt_field(self, + 'notification_configuration', + subfield=field) recipients = self.notification_configuration.pop(self.notification_class.recipient_parameter) if not isinstance(recipients, list): recipients = [recipients] diff --git a/awx/main/utils.py b/awx/main/utils.py index 5bd00c2da6..a561648f95 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -139,12 +139,13 @@ def get_encryption_key(instance, field_name): h.update(field_name) return h.digest()[:16] - -def encrypt_field(instance, field_name, ask=False): +def encrypt_field(instance, field_name, ask=False, subfield=None): ''' Return content of the given instance and field name encrypted. ''' value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] if not value or value.startswith('$encrypted$') or (ask and value == 'ASK'): return value value = smart_str(value) @@ -157,11 +158,13 @@ def encrypt_field(instance, field_name, ask=False): return '$encrypted$%s$%s' % ('AES', b64data) -def decrypt_field(instance, field_name): +def decrypt_field(instance, field_name, subfield=None): ''' Return content of the given instance and field name decrypted. ''' value = getattr(instance, field_name) + if isinstance(value, dict) and subfield is not None: + value = value[subfield] if not value or not value.startswith('$encrypted$'): return value algo, b64data = value[len('$encrypted$'):].split('$', 1) From ab3669efa9735eea8d532e1c0789453e4493765e Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 22 Feb 2016 17:09:36 -0500 Subject: [PATCH 13/23] Refactor message generator * Job object can now control the output and generate K:V output for notification types that can support it * Notifications store the body as json/dict now to encode more information * Notification Type can further compose the message based on what is sensible for the notification type * This will also allow customizing the message template in the future * All notification types use sane defaults for the level of detail now --- awx/api/serializers.py | 2 +- awx/api/views.py | 3 ++- awx/main/models/jobs.py | 20 ++++++++++++++++++++ awx/main/models/notifications.py | 17 +++++++---------- awx/main/models/unified_jobs.py | 10 ++++++++++ awx/main/notifications/email_backend.py | 7 +++++++ awx/main/notifications/hipchat_backend.py | 6 +++--- awx/main/notifications/irc_backend.py | 4 ++-- awx/main/notifications/pagerduty_backend.py | 7 +++++-- awx/main/notifications/slack_backend.py | 6 +++--- awx/main/notifications/twilio_backend.py | 6 +++--- awx/main/notifications/webhook_backend.py | 13 +++++++++---- awx/main/tasks.py | 12 ++++-------- 13 files changed, 76 insertions(+), 37 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 04f5e241c3..a680e5b00c 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2118,7 +2118,7 @@ class NotificationSerializer(BaseSerializer): class Meta: model = Notification fields = ('*', '-name', '-description', 'notifier', 'error', 'status', 'notifications_sent', - 'notification_type', 'recipients', 'subject', 'body') + 'notification_type', 'recipients', 'subject') def get_related(self, obj): res = super(NotificationSerializer, self).get_related(obj) diff --git a/awx/api/views.py b/awx/api/views.py index 439de3f845..70532f026c 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -3053,7 +3053,8 @@ class NotifierTest(GenericAPIView): def post(self, request, *args, **kwargs): obj = self.get_object() - notification = obj.generate_notification("Tower Notification Test", "Ansible Tower Test Notification") + notification = obj.generate_notification("Tower Notification Test {}".format(obj.id), + {"body": "Ansible Tower Test Notification {}".format(obj.id)}) if not notification: return Response({}, status=status.HTTP_400_BAD_REQUEST) else: diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index dd772d695d..2d2dc991a9 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -496,6 +496,26 @@ class Job(UnifiedJob, JobOptions): dependencies.append(source.create_inventory_update(launch_type='dependency')) return dependencies + def notification_data(self): + data = super(Job, self).notification_data() + all_hosts = {} + for h in self.job_host_summaries.all(): + all_hosts[h.host.name] = dict(failed=h.failed, + changed=h.changed, + dark=h.dark, + failures=h.failures, + ok=h.ok, + processed=h.processed, + skipped=h.skipped) + data.update(dict(inventory=self.inventory.name, + project=self.project.name, + playbook=self.playbook, + credential=self.credential.name, + limit=self.limit, + extra_vars=self.extra_vars, + hosts=all_hosts)) + return data + def handle_extra_data(self, extra_data): extra_vars = {} if type(extra_data) == dict: diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index d6fc9d31b8..04bd5b0e53 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -69,8 +69,8 @@ class Notifier(CommonModel): for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): if new_instance: - value = getattr(self.notification_configuration, field, '') - setattr(self, '_saved_{}'.format(field), value) + value = self.notification_configuration[field] + setattr(self, '_saved_{}_{}'.format("config", field), value) self.notification_configuration[field] = '' else: encrypted = encrypt_field(self, 'notification_configuration', subfield=field) @@ -82,8 +82,9 @@ class Notifier(CommonModel): update_fields = [] for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", self.notification_class.init_parameters): - saved_value = getattr(self, '_saved_{}'.format(field), '') - setattr(self.notification_configuration, field, saved_value) + saved_value = getattr(self, '_saved_{}_{}'.format("config", field), '') + self.notification_configuration[field] = saved_value + #setattr(self.notification_configuration, field, saved_value) if 'notification_configuration' not in update_fields: update_fields.append('notification_configuration') self.save(update_fields=update_fields) @@ -112,7 +113,7 @@ class Notifier(CommonModel): recipients = [recipients] sender = self.notification_configuration.pop(self.notification_class.sender_parameter, None) backend_obj = self.notification_class(**self.notification_configuration) - notification_obj = EmailMessage(subject, body, sender, recipients) + notification_obj = EmailMessage(subject, backend_obj.format_body(body), sender, recipients) return backend_obj.send_messages([notification_obj]) class Notification(CreatedModifiedModel): @@ -165,11 +166,7 @@ class Notification(CreatedModifiedModel): default='', editable=False, ) - body = models.TextField( - blank=True, - default='', - editable=False, - ) + body = JSONField(blank=True) def get_absolute_url(self): return reverse('api:notification_detail', args=(self.pk,)) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 986be923fb..ed34653048 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -731,6 +731,16 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique tasks that might preclude creating one''' return [] + def notification_data(self): + return dict(id=self.id, + name=self.name, + url=self.get_absolute_url(), #TODO: Need to replace with UI job view + created_by=str(self.created_by), + started=self.started.isoformat(), + finished=self.finished.isoformat(), + status=self.status, + traceback=self.result_traceback) + def start(self, error_callback, success_callback, **kwargs): ''' Start the task running via Celery. diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py index 271f585d5c..484a61f12d 100644 --- a/awx/main/notifications/email_backend.py +++ b/awx/main/notifications/email_backend.py @@ -18,3 +18,10 @@ class CustomEmailBackend(EmailBackend): recipient_parameter = "recipients" sender_parameter = "sender" + def format_body(self, body): + body_actual = "{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], + body['id'], + body['status'], + body['url']) + body_actual += pprint.pformat(body, indent=4) + return body_actual diff --git a/awx/main/notifications/hipchat_backend.py b/awx/main/notifications/hipchat_backend.py index a5b7f561b6..5d58792591 100644 --- a/awx/main/notifications/hipchat_backend.py +++ b/awx/main/notifications/hipchat_backend.py @@ -5,11 +5,11 @@ import logging import requests -from django.core.mail.backends.base import BaseEmailBackend +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.hipchat_backend') -class HipChatBackend(BaseEmailBackend): +class HipChatBackend(TowerBaseEmailBackend): init_parameters = {"token": {"label": "Token", "type": "password"}, "channels": {"label": "Destination Channels", "type": "list"}, @@ -35,7 +35,7 @@ class HipChatBackend(BaseEmailBackend): r = requests.post("{}/v2/room/{}/notification".format(self.api_url, rcp), params={"auth_token": self.token}, json={"color": self.color, - "message": m.body, + "message": m.subject, "notify": self.notify, "from": m.from_email, "message_format": "text"}) diff --git a/awx/main/notifications/irc_backend.py b/awx/main/notifications/irc_backend.py index 2b0944b74a..b3e92a12b3 100644 --- a/awx/main/notifications/irc_backend.py +++ b/awx/main/notifications/irc_backend.py @@ -7,11 +7,11 @@ import logging import irc.client -from django.core.mail.backends.base import BaseEmailBackend +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.irc_backend') -class IrcBackend(BaseEmailBackend): +class IrcBackend(TowerBaseEmailBackend): init_parameters = {"server": {"label": "IRC Server Address", "type": "string"}, "port": {"label": "IRC Server Port", "type": "int"}, diff --git a/awx/main/notifications/pagerduty_backend.py b/awx/main/notifications/pagerduty_backend.py index 161bb822bc..fd7661ba86 100644 --- a/awx/main/notifications/pagerduty_backend.py +++ b/awx/main/notifications/pagerduty_backend.py @@ -4,11 +4,11 @@ import logging import pygerduty -from django.core.mail.backends.base import BaseEmailBackend +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.pagerduty_backend') -class PagerDutyBackend(BaseEmailBackend): +class PagerDutyBackend(TowerBaseEmailBackend): init_parameters = {"subdomain": {"label": "Pagerduty subdomain", "type": "string"}, "token": {"label": "API Token", "type": "password"}, @@ -22,6 +22,9 @@ class PagerDutyBackend(BaseEmailBackend): self.subdomain = subdomain self.token = token + def format_body(self, body): + return body + def send_messages(self, messages): sent_messages = 0 diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 3bf4f32114..91e4cd4fd3 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -4,11 +4,11 @@ import logging from slackclient import SlackClient -from django.core.mail.backends.base import BaseEmailBackend +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.slack_backend') -class SlackBackend(BaseEmailBackend): +class SlackBackend(TowerBaseEmailBackend): init_parameters = {"token": {"label": "Token", "type": "password"}, "channels": {"label": "Destination Channels", "type": "list"}} @@ -41,7 +41,7 @@ class SlackBackend(BaseEmailBackend): for m in messages: try: for r in m.recipients(): - self.connection.rtm_send_message(r, m.body) + self.connection.rtm_send_message(r, m.subject) sent_messages += 1 except Exception as e: logger.error("Exception sending messages: {}".format(e)) diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index d9c4cc43b6..847ebb9f2f 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -5,11 +5,11 @@ import logging from twilio.rest import TwilioRestClient -from django.core.mail.backends.base import BaseEmailBackend +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.twilio_backend') -class TwilioBackend(BaseEmailBackend): +class TwilioBackend(TowerBaseEmailBackend): init_parameters = {"account_sid": {"label": "Account SID", "type": "string"}, "account_token": {"label": "Account Token", "type": "password"}, @@ -38,7 +38,7 @@ class TwilioBackend(BaseEmailBackend): connection.messages.create( to=m.to, from_=m.from_email, - body=m.body) + body=m.subject) sent_messages += 1 except Exception as e: logger.error("Exception sending messages: {}".format(e)) diff --git a/awx/main/notifications/webhook_backend.py b/awx/main/notifications/webhook_backend.py index 5bdbff0e02..15cd950923 100644 --- a/awx/main/notifications/webhook_backend.py +++ b/awx/main/notifications/webhook_backend.py @@ -4,12 +4,12 @@ import logging import requests - -from django.core.mail.backends.base import BaseEmailBackend +import json +from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.webhook_backend') -class WebhookBackend(BaseEmailBackend): +class WebhookBackend(TowerBaseEmailBackend): init_parameters = {"url": {"label": "Target URL", "type": "string"}, "headers": {"label": "HTTP Headers", "type": "object"}} @@ -20,11 +20,16 @@ class WebhookBackend(BaseEmailBackend): self.headers = headers super(WebhookBackend, self).__init__(fail_silently=fail_silently) + def format_body(self, body): + logger.error("Generating body from {}".format(str(body))) + return body + def send_messages(self, messages): sent_messages = 0 for m in messages: + logger.error("BODY: " + str(m.body)) r = requests.post("{}".format(m.recipients()[0]), - data=m.body, + data=json.dumps(m.body), headers=self.headers) if r.status_code >= 400: logger.error("Error sending notification webhook: {}".format(r.text)) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index aff9a6a585..7db56d78f5 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -207,10 +207,8 @@ def handle_work_success(self, result, task_actual): notification_subject = "{} #{} '{}' succeeded on Ansible Tower".format(friendly_name, task_actual['id'], instance_name) - notification_body = "{} #{} '{}' succeeded on Ansible Tower\nTo view the output: {}".format(friendly_name, - task_actual['id'], - instance_name, - instance.get_absolute_url()) + notification_body = instance.notification_data() + notification_body['friendly_name'] = friendly_name send_notifications.delay([n.generate_notification(notification_subject, notification_body) for n in notifiers.get('success', []) + notifiers.get('any', [])], job_id=task_actual['id']) @@ -265,10 +263,8 @@ def handle_work_error(self, task_id, subtasks=None): notification_subject = "{} #{} '{}' failed on Ansible Tower".format(first_task_friendly_name, first_task_id, first_task_name) - notification_body = "{} #{} '{}' failed on Ansible Tower\nTo view the output: {}".format(first_task_friendly_name, - first_task_id, - first_task_name, - first_task.get_absolute_url()) + notification_body = first_task.notification_data() + notification_body['friendly_name'] = first_task_friendly_name send_notifications.delay([n.generate_notification(notification_subject, notification_body).id for n in notifiers.get('error', []) + notifiers.get('any', [])], job_id=first_task_id) From 4928badd3fa3d5ccea6a1c7487ffcb1a22b82587 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 10:59:08 -0500 Subject: [PATCH 14/23] Add a periodic administrative notification --- awx/main/notifications/webhook_backend.py | 2 -- awx/main/tasks.py | 30 ++++++++++++++++++++--- awx/settings/defaults.py | 13 ++++++++++ 3 files changed, 39 insertions(+), 6 deletions(-) diff --git a/awx/main/notifications/webhook_backend.py b/awx/main/notifications/webhook_backend.py index 15cd950923..e10b6869e3 100644 --- a/awx/main/notifications/webhook_backend.py +++ b/awx/main/notifications/webhook_backend.py @@ -21,13 +21,11 @@ class WebhookBackend(TowerBaseEmailBackend): super(WebhookBackend, self).__init__(fail_silently=fail_silently) def format_body(self, body): - logger.error("Generating body from {}".format(str(body))) return body def send_messages(self, messages): sent_messages = 0 for m in messages: - logger.error("BODY: " + str(m.body)) r = requests.post("{}".format(m.recipients()[0]), data=json.dumps(m.body), headers=self.headers) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 7db56d78f5..4b51893a98 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -39,6 +39,8 @@ from celery import Task, task from django.conf import settings from django.db import transaction, DatabaseError from django.utils.timezone import now +from django.core.mail import send_mail +from django.contrib.auth.models import User # AWX from awx.lib.metrics import task_timer @@ -46,6 +48,7 @@ from awx.main.constants import CLOUD_PROVIDERS from awx.main.models import * # noqa from awx.main.queue import FifoQueue from awx.main.conf import tower_settings +from awx.main.task_engine import TaskSerializer, TASK_TIMEOUT_INTERVAL from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url, ignore_inventory_computed_fields, emit_websocket_notification, check_proot_installed, build_proot_temp_dir, wrap_args_with_proot) @@ -53,7 +56,7 @@ from awx.fact.utils.connection import test_mongo_connection __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate', 'RunAdHocCommand', 'handle_work_error', 'handle_work_success', - 'update_inventory_computed_fields', 'send_notifications'] + 'update_inventory_computed_fields', 'send_notifications', 'run_administrative_checks'] HIDDEN_PASSWORD = '**********' @@ -69,6 +72,8 @@ logger = logging.getLogger('awx.main.tasks') def send_notifications(notification_list, job_id=None): if not isinstance(notification_list, list): raise TypeError("notification_list should be of type list") + if job_id is not None: + job_actual = UnifiedJob.objects.get(id=job_id) for notification_id in notification_list: notification = Notification.objects.get(id=notification_id) try: @@ -82,8 +87,26 @@ def send_notifications(notification_list, job_id=None): finally: notification.save() if job_id is not None: - j = UnifiedJob.objects.get(id=job_id) - j.notifications.add(notification) + job_actual.notifications.add(notification) + +@task(bind=True) +def run_administrative_checks(self): + if not tower_settings.TOWER_ADMIN_ALERTS: + return + reader = TaskSerializer() + validation_info = reader.from_database() + used_percentage = validation_info.get('current_instances',0) / validation_info.get('instance_count', 100) + tower_admin_emails = User.objects.filter(is_superuser=True).values_list('email', flat=True) + if (used_percentage * 100) > 90: + send_mail("Ansible Tower host usage over 90%", + "Ansible Tower host usage over 90%", + tower_admin_emails, + fail_silently=True) + if validation_info.get('time_remaining', 0) < TASK_TIMEOUT_INTERVAL: + send_mail("Ansible Tower license will expire soon", + "Ansible Tower license will expire soon", + tower_admin_emails, + fail_silently=True) @task() def bulk_inventory_element_delete(inventory, hosts=[], groups=[]): @@ -155,7 +178,6 @@ def notify_task_runner(metadata_dict): queue = FifoQueue('tower_task_manager') queue.push(metadata_dict) - @task() def mongodb_control(cmd): # Sanity check: Do not send arbitrary commands. diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index d8d1fbc9fc..76ba79df61 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -341,6 +341,10 @@ CELERYBEAT_SCHEDULE = { 'task': 'awx.main.tasks.tower_periodic_scheduler', 'schedule': timedelta(seconds=30) }, + 'admin_checks': { + 'task': 'awx.main.tasks.run_administrative_checks', + 'schedule': timedelta(days=30) + }, } # Social Auth configuration. @@ -679,6 +683,8 @@ FACT_CACHE_PORT = 6564 ORG_ADMINS_CAN_SEE_ALL_USERS = True +TOWER_ADMIN_ALERTS = True + TOWER_SETTINGS_MANIFEST = { "SCHEDULE_MAX_JOBS": { "name": "Maximum Scheduled Jobs", @@ -806,6 +812,13 @@ TOWER_SETTINGS_MANIFEST = { "type": "bool", "category": "system", }, + "TOWER_ADMIN_ALERTS": { + "name": "Enable Tower Administrator Alerts", + "description": "Allow Tower to email Admin users for system events that may require attention", + "default": TOWER_ADMIN_ALERTS, + "type": "bool", + "category": "system", + }, "LICENSE": { "name": "Tower License", "description": "Controls what features and functionality is enabled in Tower.", From 39c942e98bc9e936a952c9c8c876e6fda1207b13 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 11:20:08 -0500 Subject: [PATCH 15/23] Adding migration and base notification type --- awx/main/migrations/0003_v300_changes.py | 105 +++++++++++++++++++++++ awx/main/notifications/base.py | 18 ++++ 2 files changed, 123 insertions(+) create mode 100644 awx/main/migrations/0003_v300_changes.py create mode 100644 awx/main/notifications/base.py diff --git a/awx/main/migrations/0003_v300_changes.py b/awx/main/migrations/0003_v300_changes.py new file mode 100644 index 0000000000..83b8b4b3ab --- /dev/null +++ b/awx/main/migrations/0003_v300_changes.py @@ -0,0 +1,105 @@ +# -*- coding: utf-8 -*- +from __future__ import unicode_literals + +from django.db import migrations, models +import jsonfield.fields +import django.db.models.deletion +from django.conf import settings +import taggit.managers + + +class Migration(migrations.Migration): + + dependencies = [ + ('taggit', '0002_auto_20150616_2121'), + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ('main', '0002_v300_changes'), + ] + + operations = [ + migrations.CreateModel( + name='Notification', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('created', models.DateTimeField(default=None, editable=False)), + ('modified', models.DateTimeField(default=None, editable=False)), + ('status', models.CharField(default=b'pending', max_length=20, editable=False, choices=[(b'pending', 'Pending'), (b'successful', 'Successful'), (b'failed', 'Failed')])), + ('error', models.TextField(default=b'', editable=False, blank=True)), + ('notifications_sent', models.IntegerField(default=0, editable=False)), + ('notification_type', models.CharField(max_length=32, choices=[(b'email', 'Email'), (b'slack', 'Slack'), (b'twilio', 'Twilio'), (b'pagerduty', 'Pagerduty'), (b'hipchat', 'HipChat'), (b'webhook', 'Webhook'), (b'irc', 'IRC')])), + ('recipients', models.TextField(default=b'', editable=False, blank=True)), + ('subject', models.TextField(default=b'', editable=False, blank=True)), + ('body', jsonfield.fields.JSONField(default=dict, blank=True)), + ], + options={ + 'ordering': ('pk',), + }, + ), + migrations.CreateModel( + name='Notifier', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('created', models.DateTimeField(default=None, editable=False)), + ('modified', models.DateTimeField(default=None, editable=False)), + ('description', models.TextField(default=b'', blank=True)), + ('active', models.BooleanField(default=True, editable=False)), + ('name', models.CharField(unique=True, max_length=512)), + ('notification_type', models.CharField(max_length=32, choices=[(b'email', 'Email'), (b'slack', 'Slack'), (b'twilio', 'Twilio'), (b'pagerduty', 'Pagerduty'), (b'hipchat', 'HipChat'), (b'webhook', 'Webhook'), (b'irc', 'IRC')])), + ('notification_configuration', jsonfield.fields.JSONField(default=dict)), + ('created_by', models.ForeignKey(related_name="{u'class': 'notifier', u'app_label': 'main'}(class)s_created+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)), + ('modified_by', models.ForeignKey(related_name="{u'class': 'notifier', u'app_label': 'main'}(class)s_modified+", on_delete=django.db.models.deletion.SET_NULL, default=None, editable=False, to=settings.AUTH_USER_MODEL, null=True)), + ('organization', models.ForeignKey(related_name='notifiers', on_delete=django.db.models.deletion.SET_NULL, to='main.Organization', null=True)), + ('tags', taggit.managers.TaggableManager(to='taggit.Tag', through='taggit.TaggedItem', blank=True, help_text='A comma-separated list of tags.', verbose_name='Tags')), + ], + ), + migrations.AddField( + model_name='notification', + name='notifier', + field=models.ForeignKey(related_name='notifications', editable=False, to='main.Notifier'), + ), + migrations.AddField( + model_name='activitystream', + name='notification', + field=models.ManyToManyField(to='main.Notification', blank=True), + ), + migrations.AddField( + model_name='activitystream', + name='notifier', + field=models.ManyToManyField(to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='organization', + name='notifiers_any', + field=models.ManyToManyField(related_name='organization_notifiers_for_any', to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='organization', + name='notifiers_error', + field=models.ManyToManyField(related_name='organization_notifiers_for_errors', to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='organization', + name='notifiers_success', + field=models.ManyToManyField(related_name='organization_notifiers_for_success', to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='unifiedjob', + name='notifications', + field=models.ManyToManyField(related_name='unifiedjob_notifications', editable=False, to='main.Notification'), + ), + migrations.AddField( + model_name='unifiedjobtemplate', + name='notifiers_any', + field=models.ManyToManyField(related_name='unifiedjobtemplate_notifiers_for_any', to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='unifiedjobtemplate', + name='notifiers_error', + field=models.ManyToManyField(related_name='unifiedjobtemplate_notifiers_for_errors', to='main.Notifier', blank=True), + ), + migrations.AddField( + model_name='unifiedjobtemplate', + name='notifiers_success', + field=models.ManyToManyField(related_name='unifiedjobtemplate_notifiers_for_success', to='main.Notifier', blank=True), + ), + ] diff --git a/awx/main/notifications/base.py b/awx/main/notifications/base.py new file mode 100644 index 0000000000..e574a07df3 --- /dev/null +++ b/awx/main/notifications/base.py @@ -0,0 +1,18 @@ +# Copyright (c) 2016 Ansible, Inc. +# All Rights Reserved. + +import pprint +from django.core.mail.backends.base import BaseEmailBackend + +class TowerBaseEmailBackend(BaseEmailBackend): + + def format_body(self, body): + if "body" in body: + body_actual = body['body'] + else: + body_actual = "{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], + body['id'], + body['status'], + body['url']) + body_actual += pprint.pformat(body, indent=4) + return body_actual From 050ed8a200a1ea5d53eb0e9f18c833d5e008b26e Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 12:18:31 -0500 Subject: [PATCH 16/23] Proper type for in check Still an __in for when we need to add another trigger obj --- awx/main/models/inventory.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index b0acf0a90b..fd6dfbff76 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1185,9 +1185,9 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): def notifiers(self): # Return all notifiers defined on the Project, and on the Organization for each trigger type base_notifiers = Notifier.objects.filter(active=True) - error_notifiers = list(base_notifiers.filter(organization_notifiers_for_errors__in=self)) - success_notifiers = list(base_notifiers.filter(organization_notifiers_for_success__in=self)) - any_notifiers = list(base_notifiers.filter(organization_notifiers_for_any__in=self)) + error_notifiers = list(base_notifiers.filter(organization_notifiers_for_errors__in=[self])) + success_notifiers = list(base_notifiers.filter(organization_notifiers_for_success__in=[self])) + any_notifiers = list(base_notifiers.filter(organization_notifiers_for_any__in=[self])) return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) def clean_source(self): From b88892be49dca2db9d7a16e2696f6da6cedeb9b1 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 12:33:30 -0500 Subject: [PATCH 17/23] Sanity check and force proper types in admin check --- awx/main/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 4b51893a98..58e5425866 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -95,7 +95,9 @@ def run_administrative_checks(self): return reader = TaskSerializer() validation_info = reader.from_database() - used_percentage = validation_info.get('current_instances',0) / validation_info.get('instance_count', 100) + if validation_info.get('instance_count', 0) < 1: + return + used_percentage = float(validation_info.get('current_instances', 0)) / float(validation_info.get('instance_count', 100)) tower_admin_emails = User.objects.filter(is_superuser=True).values_list('email', flat=True) if (used_percentage * 100) > 90: send_mail("Ansible Tower host usage over 90%", From 75ef0dd395fc08ef6e87a8fc03d8d90310736c08 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 14:19:42 -0500 Subject: [PATCH 18/23] Implement tower ui view url on models --- awx/api/views.py | 5 +++-- awx/main/models/ad_hoc_commands.py | 4 ++++ awx/main/models/inventory.py | 5 +++++ awx/main/models/jobs.py | 7 +++++++ awx/main/models/projects.py | 4 ++++ awx/main/models/unified_jobs.py | 9 ++++++++- awx/main/tasks.py | 15 ++++++++------- awx/settings/defaults.py | 9 +++++++++ 8 files changed, 48 insertions(+), 10 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 70532f026c..e5975c7173 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -268,6 +268,7 @@ class ApiV1ConfigView(APIView): # If the license is valid, write it to disk. if license_data['valid_key']: tower_settings.LICENSE = data_actual + tower_settings.TOWER_URL_BASE = "{}://{}".format(request.scheme, request.get_host()) # Spawn a task to ensure that MongoDB is started (or stopped) # as appropriate, based on whether the license uses it. @@ -3053,8 +3054,8 @@ class NotifierTest(GenericAPIView): def post(self, request, *args, **kwargs): obj = self.get_object() - notification = obj.generate_notification("Tower Notification Test {}".format(obj.id), - {"body": "Ansible Tower Test Notification {}".format(obj.id)}) + notification = obj.generate_notification("Tower Notification Test {} {}".format(obj.id, tower_settings.TOWER_URL_BASE), + {"body": "Ansible Tower Test Notification {} {}".format(obj.id, tower_settings.TOWER_URL_BASE)}) if not notification: return Response({}, status=status.HTTP_400_BAD_REQUEST) else: diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 664269a188..12c4261d8b 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -5,6 +5,7 @@ import hmac import json import logging +from urlparse import urljoin # Django from django.conf import settings @@ -139,6 +140,9 @@ class AdHocCommand(UnifiedJob): def get_absolute_url(self): return reverse('api:ad_hoc_command_detail', args=(self.pk,)) + def get_ui_url(self): + return urljoin(tower_settings.TOWER_URL_BASE, "/#/ad_hoc_commands/{}".format(self.pk)) + @property def task_auth_token(self): '''Return temporary auth token used for task requests via API.''' diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index fd6dfbff76..edf03a883d 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -6,6 +6,7 @@ import datetime import logging import re import copy +from urlparse import urljoin # Django from django.conf import settings @@ -25,6 +26,7 @@ from awx.main.models.jobs import Job from awx.main.models.unified_jobs import * # noqa from awx.main.models.notifications import Notifier from awx.main.utils import ignore_inventory_computed_fields, _inventory_updates +from awx.main.conf import tower_settings __all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript'] @@ -1249,6 +1251,9 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions): def get_absolute_url(self): return reverse('api:inventory_update_detail', args=(self.pk,)) + def get_ui_url(self): + return urljoin(tower_settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk)) + def is_blocked_by(self, obj): if type(obj) == InventoryUpdate: if self.inventory_source.inventory == obj.inventory_source.inventory: diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 2d2dc991a9..01857b8b06 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -6,6 +6,7 @@ import hmac import json import yaml import logging +from urlparse import urljoin # Django from django.conf import settings @@ -380,6 +381,9 @@ class Job(UnifiedJob, JobOptions): def get_absolute_url(self): return reverse('api:job_detail', args=(self.pk,)) + def get_ui_url(self): + return urljoin(tower_settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk)) + @property def task_auth_token(self): '''Return temporary auth token used for task requests via API.''' @@ -1096,6 +1100,9 @@ class SystemJob(UnifiedJob, SystemJobOptions): def get_absolute_url(self): return reverse('api:system_job_detail', args=(self.pk,)) + def get_ui_url(self): + return urljoin(tower_settings.TOWER_URL_BASE, "/#/management_jobs/{}".format(self.pk)) + def is_blocked_by(self, obj): return True diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 8a320e3cfc..01e4220d6d 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -24,6 +24,7 @@ from awx.main.models.jobs import Job from awx.main.models.notifications import Notifier from awx.main.models.unified_jobs import * # noqa from awx.main.utils import update_scm_url +from awx.main.conf import tower_settings __all__ = ['Project', 'ProjectUpdate'] @@ -389,6 +390,9 @@ class ProjectUpdate(UnifiedJob, ProjectOptions): def get_absolute_url(self): return reverse('api:project_update_detail', args=(self.pk,)) + def get_ui_url(self): + return urlparse.urljoin(tower_settings.TOWER_URL_BASE, "/#/scm_update/{}".format(self.pk)) + def _update_parent_instance(self): parent_instance = self._get_parent_instance() if parent_instance: diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index ed34653048..9a324048c3 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -484,6 +484,13 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique else: return '' + def get_ui_url(self): + real_instance = self.get_real_instance() + if real_instance != self: + return real_instance.get_ui_url() + else: + return '' + @classmethod def _get_task_class(cls): raise NotImplementedError # Implement in subclasses. @@ -734,7 +741,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique def notification_data(self): return dict(id=self.id, name=self.name, - url=self.get_absolute_url(), #TODO: Need to replace with UI job view + url=self.get_ui_url(), created_by=str(self.created_by), started=self.started.isoformat(), finished=self.finished.isoformat(), diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 58e5425866..ee65490ec1 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -228,11 +228,11 @@ def handle_work_success(self, result, task_actual): friendly_name = "AdHoc Command" else: return - notification_subject = "{} #{} '{}' succeeded on Ansible Tower".format(friendly_name, - task_actual['id'], - instance_name) notification_body = instance.notification_data() - notification_body['friendly_name'] = friendly_name + notification_subject = "{} #{} '{}' succeeded on Ansible Tower: {}".format(friendly_name, + task_actual['id'], + instance_name, + notification_body['url']) send_notifications.delay([n.generate_notification(notification_subject, notification_body) for n in notifiers.get('success', []) + notifiers.get('any', [])], job_id=task_actual['id']) @@ -284,10 +284,11 @@ def handle_work_error(self, task_id, subtasks=None): (first_task_type, first_task_name, first_task_id) instance.save() instance.socketio_emit_status("failed") - notification_subject = "{} #{} '{}' failed on Ansible Tower".format(first_task_friendly_name, - first_task_id, - first_task_name) notification_body = first_task.notification_data() + notification_subject = "{} #{} '{}' failed on Ansible Tower: {}".format(first_task_friendly_name, + first_task_id, + first_task_name, + notification_body['url']) notification_body['friendly_name'] = first_task_friendly_name send_notifications.delay([n.generate_notification(notification_subject, notification_body).id for n in notifiers.get('error', []) + notifiers.get('any', [])], diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 76ba79df61..465809b523 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -685,6 +685,8 @@ ORG_ADMINS_CAN_SEE_ALL_USERS = True TOWER_ADMIN_ALERTS = True +TOWER_URL_BASE = "https://towerhost" + TOWER_SETTINGS_MANIFEST = { "SCHEDULE_MAX_JOBS": { "name": "Maximum Scheduled Jobs", @@ -819,6 +821,13 @@ TOWER_SETTINGS_MANIFEST = { "type": "bool", "category": "system", }, + "TOWER_URL_BASE": { + "name": "Base URL of the Tower host", + "description": "This is used by services like Notifications to render a valid url to the Tower host", + "default": TOWER_URL_BASE, + "type": "string", + "category": "system", + }, "LICENSE": { "name": "Tower License", "description": "Controls what features and functionality is enabled in Tower.", From 4b1493f456ea1f1327f12692c20d4d135ad57c4d Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 15:26:29 -0500 Subject: [PATCH 19/23] Fixing up some unicode issues --- awx/main/notifications/base.py | 10 ++++++---- awx/main/notifications/email_backend.py | 9 +++++---- awx/main/notifications/hipchat_backend.py | 6 ++++-- awx/main/notifications/irc_backend.py | 4 +++- awx/main/notifications/pagerduty_backend.py | 6 ++++-- awx/main/notifications/slack_backend.py | 4 +++- awx/main/notifications/twilio_backend.py | 6 ++++-- awx/main/notifications/webhook_backend.py | 8 +++++--- awx/main/tasks.py | 3 ++- 9 files changed, 36 insertions(+), 20 deletions(-) diff --git a/awx/main/notifications/base.py b/awx/main/notifications/base.py index e574a07df3..8129c33e27 100644 --- a/awx/main/notifications/base.py +++ b/awx/main/notifications/base.py @@ -2,6 +2,8 @@ # All Rights Reserved. import pprint + +from django.utils.encoding import smart_text from django.core.mail.backends.base import BaseEmailBackend class TowerBaseEmailBackend(BaseEmailBackend): @@ -10,9 +12,9 @@ class TowerBaseEmailBackend(BaseEmailBackend): if "body" in body: body_actual = body['body'] else: - body_actual = "{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], - body['id'], - body['status'], - body['url']) + body_actual = smart_text("{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], + body['id'], + body['status'], + body['url'])) body_actual += pprint.pformat(body, indent=4) return body_actual diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py index 484a61f12d..364e45fc28 100644 --- a/awx/main/notifications/email_backend.py +++ b/awx/main/notifications/email_backend.py @@ -3,6 +3,7 @@ import logging +from django.utils.encoding import smart_text from django.core.mail.backends.smtp import EmailBackend class CustomEmailBackend(EmailBackend): @@ -19,9 +20,9 @@ class CustomEmailBackend(EmailBackend): sender_parameter = "sender" def format_body(self, body): - body_actual = "{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], - body['id'], - body['status'], - body['url']) + body_actual = smart_text("{} #{} had status {} on Ansible Tower, view details at {}\n\n".format(body['friendly_name'], + body['id'], + body['status'], + body['url'])) body_actual += pprint.pformat(body, indent=4) return body_actual diff --git a/awx/main/notifications/hipchat_backend.py b/awx/main/notifications/hipchat_backend.py index 5d58792591..420ef928fa 100644 --- a/awx/main/notifications/hipchat_backend.py +++ b/awx/main/notifications/hipchat_backend.py @@ -5,6 +5,8 @@ import logging import requests +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.hipchat_backend') @@ -40,8 +42,8 @@ class HipChatBackend(TowerBaseEmailBackend): "from": m.from_email, "message_format": "text"}) if r.status_code != 204: - logger.error("Error sending messages: {}".format(r.text)) + logger.error(smart_text("Error sending messages: {}".format(r.text))) if not self.fail_silently: - raise Exception("Error sending message to hipchat: {}".format(r.text)) + raise Exception(smart_text("Error sending message to hipchat: {}".format(r.text))) sent_messages += 1 return sent_messages diff --git a/awx/main/notifications/irc_backend.py b/awx/main/notifications/irc_backend.py index b3e92a12b3..e01d390c09 100644 --- a/awx/main/notifications/irc_backend.py +++ b/awx/main/notifications/irc_backend.py @@ -7,6 +7,8 @@ import logging import irc.client +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.irc_backend') @@ -48,7 +50,7 @@ class IrcBackend(TowerBaseEmailBackend): connect_factory=connection_factory, ) except irc.client.ServerConnectionError as e: - logger.error("Exception connecting to irc server: {}".format(e)) + logger.error(smart_text("Exception connecting to irc server: {}".format(e))) if not self.fail_silently: raise return True diff --git a/awx/main/notifications/pagerduty_backend.py b/awx/main/notifications/pagerduty_backend.py index fd7661ba86..af6b95cfd6 100644 --- a/awx/main/notifications/pagerduty_backend.py +++ b/awx/main/notifications/pagerduty_backend.py @@ -4,6 +4,8 @@ import logging import pygerduty +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.pagerduty_backend') @@ -33,7 +35,7 @@ class PagerDutyBackend(TowerBaseEmailBackend): except Exception as e: if not self.fail_silently: raise - logger.error("Exception connecting to PagerDuty: {}".format(e)) + logger.error(smart_text("Exception connecting to PagerDuty: {}".format(e))) for m in messages: try: pager.trigger_incident(m.recipients()[0], @@ -41,7 +43,7 @@ class PagerDutyBackend(TowerBaseEmailBackend): details=m.body, client=m.from_email) except Exception as e: - logger.error("Exception sending messages: {}".format(e)) + logger.error(smart_text("Exception sending messages: {}".format(e))) if not self.fail_silently: raise return sent_messages diff --git a/awx/main/notifications/slack_backend.py b/awx/main/notifications/slack_backend.py index 91e4cd4fd3..00f23ed60c 100644 --- a/awx/main/notifications/slack_backend.py +++ b/awx/main/notifications/slack_backend.py @@ -4,6 +4,8 @@ import logging from slackclient import SlackClient +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.slack_backend') @@ -44,7 +46,7 @@ class SlackBackend(TowerBaseEmailBackend): self.connection.rtm_send_message(r, m.subject) sent_messages += 1 except Exception as e: - logger.error("Exception sending messages: {}".format(e)) + logger.error(smart_text("Exception sending messages: {}".format(e))) if not self.fail_silently: raise return sent_messages diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index 847ebb9f2f..1aea6f368e 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -5,6 +5,8 @@ import logging from twilio.rest import TwilioRestClient +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.twilio_backend') @@ -31,7 +33,7 @@ class TwilioBackend(TowerBaseEmailBackend): except Exception as e: if not self.fail_silently: raise - logger.error("Exception connecting to Twilio: {}".format(e)) + logger.error(smart_text("Exception connecting to Twilio: {}".format(e))) for m in messages: try: @@ -41,7 +43,7 @@ class TwilioBackend(TowerBaseEmailBackend): body=m.subject) sent_messages += 1 except Exception as e: - logger.error("Exception sending messages: {}".format(e)) + logger.error(smart_text("Exception sending messages: {}".format(e))) if not self.fail_silently: raise return sent_messages diff --git a/awx/main/notifications/webhook_backend.py b/awx/main/notifications/webhook_backend.py index e10b6869e3..52d85483ab 100644 --- a/awx/main/notifications/webhook_backend.py +++ b/awx/main/notifications/webhook_backend.py @@ -2,9 +2,11 @@ # All Rights Reserved. import logging - import requests import json + +from django.utils.encoding import smart_text + from awx.main.notifications.base import TowerBaseEmailBackend logger = logging.getLogger('awx.main.notifications.webhook_backend') @@ -30,8 +32,8 @@ class WebhookBackend(TowerBaseEmailBackend): data=json.dumps(m.body), headers=self.headers) if r.status_code >= 400: - logger.error("Error sending notification webhook: {}".format(r.text)) + logger.error(smart_text("Error sending notification webhook: {}".format(r.text))) if not self.fail_silently: - raise Exception("Error sending notification webhook: {}".format(r.text)) + raise Exception(smart_text("Error sending notification webhook: {}".format(r.text))) sent_messages += 1 return sent_messages diff --git a/awx/main/tasks.py b/awx/main/tasks.py index ee65490ec1..4b285546bb 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -39,6 +39,7 @@ from celery import Task, task from django.conf import settings from django.db import transaction, DatabaseError from django.utils.timezone import now +from django.utils.encoding import smart_text from django.core.mail import send_mail from django.contrib.auth.models import User @@ -83,7 +84,7 @@ def send_notifications(notification_list, job_id=None): except Exception as e: logger.error("Send Notification Failed {}".format(e)) notification.status = "failed" - notification.error = str(e) + notification.error = smart_text(e) finally: notification.save() if job_id is not None: From 41d5393af03e29989897b82844ce20b5835b3c27 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Tue, 23 Feb 2016 15:30:07 -0500 Subject: [PATCH 20/23] Clean up flake8 related issues --- awx/api/serializers.py | 2 +- awx/main/models/notifications.py | 2 +- awx/main/models/projects.py | 1 - awx/main/models/unified_jobs.py | 1 - awx/main/notifications/email_backend.py | 2 +- awx/main/notifications/irc_backend.py | 2 +- 6 files changed, 4 insertions(+), 6 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index a680e5b00c..f42c20812c 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -2108,7 +2108,7 @@ class NotifierSerializer(BaseSerializer): if incorrect_type_fields: for type_field_error in incorrect_type_fields: error_list.append("Configuration field '{}' incorrect type, expected {}".format(type_field_error[0], - type_field_error[1])) + type_field_error[1])) if error_list: raise serializers.ValidationError(error_list) return attrs diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index 04bd5b0e53..29a51cf9ac 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -104,7 +104,7 @@ class Notifier(CommonModel): def send(self, subject, body): for field in filter(lambda x: self.notification_class.init_parameters[x]['type'] == "password", - self.notification_class.init_parameters): + self.notification_class.init_parameters): self.notification_configuration[field] = decrypt_field(self, 'notification_configuration', subfield=field) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 01e4220d6d..415c674bb1 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -10,7 +10,6 @@ import urlparse # Django from django.conf import settings from django.db import models -from django.db.models import Q from django.utils.translation import ugettext_lazy as _ from django.utils.encoding import smart_str, smart_text from django.core.exceptions import ValidationError diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 9a324048c3..7bb4cdd798 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -30,7 +30,6 @@ from djcelery.models import TaskMeta # AWX from awx.main.models.base import * # noqa from awx.main.models.schedules import Schedule -from awx.main.models.notifications import Notification from awx.main.utils import decrypt_field, emit_websocket_notification, _inventory_updates from awx.main.redact import UriCleaner diff --git a/awx/main/notifications/email_backend.py b/awx/main/notifications/email_backend.py index 364e45fc28..9a9d0a9e2d 100644 --- a/awx/main/notifications/email_backend.py +++ b/awx/main/notifications/email_backend.py @@ -1,7 +1,7 @@ # Copyright (c) 2016 Ansible, Inc. # All Rights Reserved. -import logging +import pprint from django.utils.encoding import smart_text from django.core.mail.backends.smtp import EmailBackend diff --git a/awx/main/notifications/irc_backend.py b/awx/main/notifications/irc_backend.py index e01d390c09..61158bbe5d 100644 --- a/awx/main/notifications/irc_backend.py +++ b/awx/main/notifications/irc_backend.py @@ -88,7 +88,7 @@ class IrcBackend(TowerBaseEmailBackend): self.connection.add_global_handler("join", self.on_join) start_time = time.time() process_time = time.time() - while self.channels_sent < len(self.channels) and (process_time-start_time) < 60: + while self.channels_sent < len(self.channels) and (process_time - start_time) < 60: self.reactor.process_once(0.1) process_time = time.time() self.reactor.disconnect_all() From b35d7a3c6bd86ee09ff0ad563814f7375e73495d Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Wed, 24 Feb 2016 13:26:43 -0500 Subject: [PATCH 21/23] Add notification system documentation --- docs/notification_system.md | 187 ++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 docs/notification_system.md diff --git a/docs/notification_system.md b/docs/notification_system.md new file mode 100644 index 0000000000..fc8f99b9ee --- /dev/null +++ b/docs/notification_system.md @@ -0,0 +1,187 @@ +Completion pending unit tests and acceptance info and instructions. The following documentation will likely be moved to the feature epic card and reproduced in our development documentation. + +# Notification System Overview + +A Notifier is an instance of a notification type (Email, Slack, Webhook, etc) with a name, description, and a defined configuration (A few examples: Username, password, server, recipients for the Email type. Token and list of channels for Slack. Url and Headers for webhooks) + +A Notification is a manifestation of the Notifier... for example, when a job fails a notification is sent using the configuration defined by the Notifier. + +This PR implements the Notification system as outlined in the 3.0 Notifications spec. At a high level the typical flow is: + +* User creates a Notifier at `/api/v1/notifiers` +* User assigns the notifier to any of the various objects that support it (all variants of job templates as well as organizations and projects) and at the appropriate trigger level for which they want the notification (error, success, or any). For example a user may wish to assign a particular Notifier to trigger when `Job Template 1` fails. In which case they will associate the notifier with the job template at `/api/v1/job_templates/n/notifiers_error`. + +## Notifier hierarchy + +Notifiers assigned at certain levels will inherit notifiers defined on parent objects as such: + +* Job Templates will use notifiers defined on it as well as inheriting notifiers from the Project used by the Job Template and from the Organization that it is listed under (via the Project). +* Project Updates will use notifiers defined on the project and will inherit notifiers from the Organization associated with it. +* Inventory Updates will use notifiers defined on the Organization that it is listed under +* Ad-hoc commands will use notifiers defined on the Organization that the inventory is associated with + +## Workflow + +When a job succeeds or fails, the error or success handler will pull a list of relevant notifiers using the procedure defined above. It will then create a Notification object for each one containing relevant details about the job and then **send**s it to the destination (email addresses, slack channel(s), sms numbers, etc). These Notification objects are available as related resources on job types (jobs, inventory updates, project updates), and also at `/api/v1/notifications`. You may also see what notifications have been sent from a notifier by examining its related resources. + +Notifications can succeed or fail but that will not cause its associated job to succeed or fail. The status of the notification can be viewed at its detail endpoint `/api/v1/notifications/` + +## Testing Notifiers before using them + +Once a Notifier is created its configuration can be tested by utilizing the endpoint at `/api/v1/notifiers//test` This will emit a test notification given the configuration defined by the Notifier. These test notifications will also appear in the notifications list at `/api/v1/notifications` + +# Notification Types + +The currently defined Notification Types are: + +* Email +* Slack +* Hipchat +* Pagerduty +* Twilio +* IRC +* Webhook + +Each of these have their own configuration and behavioral semantics and testing them may need to be approached in different ways. The following sections will give as much detail as possible. + +## Email + +The email notification type supports a wide variety of smtp servers and has support for ssl/tls connections. + +### Testing considerations + +The following should be performed for good acceptance: + +* Test plain authentication +* Test SSL and TLS authentication +* Verify single and multiple recipients +* Verify message subject and contents are formatted sanely. They should be plaintext but readable. + +### Test Service + +Either setup a local smtp mail service here are some options: + +* postfix service on galaxy: https://galaxy.ansible.com/debops/postfix/ +* Mailtrap has a good free plan and should provide all of the features we need under that plan: https://mailtrap.io/ + +## Slack + +Slack is pretty easy to configure, it just needs a token which you can get from creating a bot in the integrations settings for the slack team. + +### Testing considerations + +The following should be performed for good acceptance: + +* Test single and multiple channels and good formatting of the message. Note that slack notifications only contain the minimal information + +### Test Service + +Any user of the Ansible slack service can create a bot integration (which is how this notification is implemented). Remember to invite the bot to the channel first. + +## Hipchat + +There are several ways to integrate with hipchat. The Tower implementation uses Hipchat "Integrations". Currently you can find this at the bottom right of the main hipchat webview. From there you will select "Build your own Integration". After creating that it will list the `auth_token` that needs to be supplied to Tower. Some other relevant details on the fields accepted by Tower for the Hipchat notification type: + +* `color`: This will highlight the message as the given color. If set to something hipchat doesn't expect then the notification will generate an error, but it's pretty rad. I like green personally. +* `notify`: Selecting this will cause the bot to "notify" channel members. Normally it will just be stuck as a message in the chat channel without triggering anyone's notifications. This option will notify users of the channel respecting their existing notification settings (browser notification, email fallback, etc.) +* `message_from`: Along with the integration name itself this will put another label on the notification. I reckon this would be helpful if multiple services are using the same integration to distinguish them from each other. +* `api_url`: The url of the hipchat api service. If you create a team hosted by them it'll be something like `https://team.hipchat.com`. For a self-hosted service it'll be the http url that is accessible by Tower. + +### Testing considerations + +* Make sure all options behave as expected +* Test single and multiple channels +* Test that notification preferences are obeyed. +* Test formatting and appearance. Note that, like Slack, hipchat will use the minimal version of the notification. +* Test standalone hipchat service for parity with hosted solution + +### Test Service + +Hipchat allows you to create a team with limited users and message history for free, which is easy to set up and get started with. Hipchat contains a self-hosted server also which we should test for parity... it has a 30 day trial but there might be some other way to negotiate with them, redhat, or ansible itself: + +https://www.hipchat.com/server + +## Pagerduty + +Pager duty is a fairly straightforward integration. The user will create an API Key in the pagerduty system (this will be the token that is given to Tower) and then create a "Service" which will provide an "Integration Key" that will be given to Tower also. The other options of note are: + +* `subdomain`: When you sign up for the pagerduty account you will get a unique subdomain to communicate with. For instance, if you signed up as "towertest" the web dashboard will be at towertest.pagerduty.com and you will give the Tower API "towertest" as the subdomain (not the full domain). +* `client_name`: This will be sent along with the alert content to the pagerduty service to help identify the service that is using the api key/service. This is helpful if multiple integrations are using the same api key and service. + +### Testing considerations + +* Make sure the alert lands on the pagerduty service +* Verify that the minimal information is displayed for the notification but also that the detail of the notification contains all fields. Pagerduty itself should understand the format in which we send the detail information. + +### Test Service + +Pagerduty allows you to sign up for a free trial with the service. We may also have a ansible-wide pagerduty service that we could tie into for other things. + +## Twilio + +Twilio service is an Voice and SMS automation service. Once you are signed in you'll need to create a phone number from which the message will be sent. You'll then define a "Messaging Service" under Programmable SMS and associate the number you created before with it. Note that you may need to verify this number or some other information before you are allowed to use it to send to any numbers. The Messaging Service does not need a status callback url nor does it need the ability to Process inbound messages. + +Under your individual (or sub) account settings you will have API credentials. The Account SID and AuthToken are what will be given to Tower. There are a couple of other important fields: + +* `from_number`: This is the number associated with the messaging service above and must be given in the form of "+15556667777" +* `to_numbers`: This will be the list of numbers to receive the SMS and should be the 10-digit phone number. + +### Testing considerations + +* Test notifications with single and multiple recipients +* Verify that the minimal information is displayed for the notification. Note that this notification type does not display the full detailed notification. + +### Test Service + +Twilio is fairly straightforward to sign up for but I don't believe it has a free plan, a credit card will be needed to sign up for it though the charges are fairly minimal per message. + +## IRC + +The Tower irc notification takes the form of an IRC bot that will connect, deliver its messages to channel(s) or individual user(s), and then disconnect. The Tower notification bot also supports SSL authentication. The Tower bot does not currently support Nickserv identification. If a channel or user does not exist or is not on-line then the Notification will not fail, the failure scenario is reserved specifically for connectivity. + +Connectivity information is straightforward: + +* `server`: The host name or address of the irc server +* `port`: The irc server port +* `nickname`: The bot's nickname once it connects to the server +* `password`: IRC servers can require a password to connect. If the server doesn't require one then this should be an empty string +* `use_ssl`: Should the bot use SSL when connecting +* `targets`: A list of users and/or channels to send the notification to. + +### Test Considerations + +* Test both plain and SSL connectivity +* Test single and multiples of both users and channels. + +### Test Service + +There are a few modern irc servers to choose from but we should use a fairly full featured service to get good test coverage. I recommend inspircd because it is actively maintained and pretty straightforward to configure. + +## Webhook + +The webhook notification type in Ansible Tower provides a simple interface to sending POSTs to a predefined web service. Tower will POST to this address using `application/json` content type with the data payload containing all relevant details in json format. +The parameters are pretty straightforward: + +* `url`: The full url that will be POSTed to +* `headers`: Headers in json form where the keys and values are strings. For example: `{"Authentication": "988881adc9fc3655077dc2d4d757d480b5ea0e11", "MessageType": "Test"}` + +### Test Considerations + +* Test HTTP service and HTTPS, also specifically test HTTPS with a self signed cert. +* Verify that the headers and payload are present and that the payload is json and the content type is specifically `application/json` + +### Test Service + +A very basic test can be performed by using `netcat`: + +``` +netcat -l 8099 +``` + +and then sending the request to: http://\:8099 + +Note that this won't respond correctly to the notification so it will yield an error. I recommend using a very basic Flask application for verifying the POST request, you can see an example of mine here: + +https://gist.github.com/matburt/73bfbf85c2443f39d272 + +This demonstrates how to define an endpoint and parse headers and json content, it doesn't show configuring Flask for HTTPS but this is also pretty straightforward: http://flask.pocoo.org/snippets/111/ From 0ee12901fe4e5927f2c9d8e5f52a30223f9b5cb5 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 29 Feb 2016 12:30:00 -0500 Subject: [PATCH 22/23] Fix some notifications issues and write some tests * Fixes some notifier merging issues * Fixes some more unicode problems * Implements unit tests --- awx/main/models/inventory.py | 7 +- awx/main/models/jobs.py | 6 +- awx/main/models/projects.py | 15 +- awx/main/models/unified_jobs.py | 3 +- awx/main/notifications/twilio_backend.py | 3 +- awx/main/tasks.py | 4 +- awx/main/tests/functional/conftest.py | 160 ++++++++++++++++++ .../tests/functional/test_notifications.py | 124 ++++++++++++++ 8 files changed, 302 insertions(+), 20 deletions(-) create mode 100644 awx/main/tests/functional/conftest.py create mode 100644 awx/main/tests/functional/test_notifications.py diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index edf03a883d..c95c8488bd 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -1185,11 +1185,10 @@ class InventorySource(UnifiedJobTemplate, InventorySourceOptions): @property def notifiers(self): - # Return all notifiers defined on the Project, and on the Organization for each trigger type base_notifiers = Notifier.objects.filter(active=True) - error_notifiers = list(base_notifiers.filter(organization_notifiers_for_errors__in=[self])) - success_notifiers = list(base_notifiers.filter(organization_notifiers_for_success__in=[self])) - any_notifiers = list(base_notifiers.filter(organization_notifiers_for_any__in=[self])) + error_notifiers = list(base_notifiers.filter(organization_notifiers_for_errors=self.inventory.organization)) + success_notifiers = list(base_notifiers.filter(organization_notifiers_for_success=self.inventory.organization)) + any_notifiers = list(base_notifiers.filter(organization_notifiers_for_any=self.inventory.organization)) return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) def clean_source(self): diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 01857b8b06..bd167d3474 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -341,7 +341,11 @@ class JobTemplate(UnifiedJobTemplate, JobOptions): error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_errors__in=[self, self.project])) success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_success__in=[self, self.project])) any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_any__in=[self, self.project])) - return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) + # Get Organization Notifiers + error_notifiers = set(error_notifiers + list(base_notifiers.filter(organization_notifiers_for_errors__in=self.project.organizations.all()))) + success_notifiers = set(success_notifiers + list(base_notifiers.filter(organization_notifiers_for_success__in=self.project.organizations.all()))) + any_notifiers = set(any_notifiers + list(base_notifiers.filter(organization_notifiers_for_any__in=self.project.organizations.all()))) + return dict(error=list(error_notifiers), success=list(success_notifiers), any=list(any_notifiers)) class Job(UnifiedJob, JobOptions): ''' diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 415c674bb1..db295023da 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -313,20 +313,15 @@ class Project(UnifiedJobTemplate, ProjectOptions): @property def notifiers(self): - # Return all notifiers defined on the Project, and on the Organization for each trigger type - # TODO: Currently there is no org fk on project so this will need to be added back once that is - # available after the rbac pr base_notifiers = Notifier.objects.filter(active=True) - # error_notifiers = list(base_notifiers.filter(Q(project_notifications_for_errors__in=self) | - # Q(organization_notifications_for_errors__in=self.organization))) - # success_notifiers = list(base_notifiers.filter(Q(project_notifications_for_success__in=self) | - # Q(organization_notifications_for_success__in=self.organization))) - # any_notifiers = list(base_notifiers.filter(Q(project_notifications_for_any__in=self) | - # Q(organization_notifications_for_any__in=self.organization))) error_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_errors=self)) success_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_success=self)) any_notifiers = list(base_notifiers.filter(unifiedjobtemplate_notifiers_for_any=self)) - return dict(error=error_notifiers, success=success_notifiers, any=any_notifiers) + # Get Organization Notifiers + error_notifiers = set(error_notifiers + list(base_notifiers.filter(organization_notifiers_for_errors__in=self.organizations.all()))) + success_notifiers = set(success_notifiers + list(base_notifiers.filter(organization_notifiers_for_success__in=self.organizations.all()))) + any_notifiers = set(any_notifiers + list(base_notifiers.filter(organization_notifiers_for_any__in=self.organizations.all()))) + return dict(error=list(error_notifiers), success=list(success_notifiers), any=list(any_notifiers)) def get_absolute_url(self): return reverse('api:project_detail', args=(self.pk,)) diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 7bb4cdd798..d83cfae978 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -17,6 +17,7 @@ from django.db import models from django.core.exceptions import NON_FIELD_ERRORS from django.utils.translation import ugettext_lazy as _ from django.utils.timezone import now +from django.utils.encoding import smart_text # Django-JSONField from jsonfield import JSONField @@ -741,7 +742,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique return dict(id=self.id, name=self.name, url=self.get_ui_url(), - created_by=str(self.created_by), + created_by=smart_text(self.created_by), started=self.started.isoformat(), finished=self.finished.isoformat(), status=self.status, diff --git a/awx/main/notifications/twilio_backend.py b/awx/main/notifications/twilio_backend.py index 1aea6f368e..df411c68c5 100644 --- a/awx/main/notifications/twilio_backend.py +++ b/awx/main/notifications/twilio_backend.py @@ -20,11 +20,10 @@ class TwilioBackend(TowerBaseEmailBackend): recipient_parameter = "to_numbers" sender_parameter = "from_number" - def __init__(self, account_sid, account_token, from_phone, fail_silently=False, **kwargs): + def __init__(self, account_sid, account_token, fail_silently=False, **kwargs): super(TwilioBackend, self).__init__(fail_silently=fail_silently) self.account_sid = account_sid self.account_token = account_token - self.from_phone = from_phone def send_messages(self, messages): sent_messages = 0 diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 4b285546bb..509c5d1e7e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -235,7 +235,7 @@ def handle_work_success(self, result, task_actual): instance_name, notification_body['url']) send_notifications.delay([n.generate_notification(notification_subject, notification_body) - for n in notifiers.get('success', []) + notifiers.get('any', [])], + for n in set(notifiers.get('success', []) + notifiers.get('any', []))], job_id=task_actual['id']) @task(bind=True) @@ -292,7 +292,7 @@ def handle_work_error(self, task_id, subtasks=None): notification_body['url']) notification_body['friendly_name'] = first_task_friendly_name send_notifications.delay([n.generate_notification(notification_subject, notification_body).id - for n in notifiers.get('error', []) + notifiers.get('any', [])], + for n in set(notifiers.get('error', []) + notifiers.get('any', []))], job_id=first_task_id) diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py new file mode 100644 index 0000000000..27ff76f816 --- /dev/null +++ b/awx/main/tests/functional/conftest.py @@ -0,0 +1,160 @@ +import pytest +import mock + +from django.core.urlresolvers import resolve +from django.utils.six.moves.urllib.parse import urlparse + +from awx.main.models.organization import Organization +from awx.main.models.projects import Project +from awx.main.models.ha import Instance +from django.contrib.auth.models import User +from rest_framework.test import ( + APIRequestFactory, + force_authenticate, +) + +@pytest.fixture +def user(): + def u(name, is_superuser=False): + try: + user = User.objects.get(username=name) + except User.DoesNotExist: + user = User(username=name, is_superuser=is_superuser, password=name) + user.save() + return user + return u + +@pytest.fixture +def post(): + def rf(url, data, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().post(url, data, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def get(): + def rf(url, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().get(url, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def put(): + def rf(url, data, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().put(url, data, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def patch(): + def rf(url, data, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().patch(url, data, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def delete(): + def rf(url, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().delete(url, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def head(): + def rf(url, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().head(url, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def options(): + def rf(url, data, user=None, middleware=None, **kwargs): + view, view_args, view_kwargs = resolve(urlparse(url)[2]) + if 'format' not in kwargs: + kwargs['format'] = 'json' + request = APIRequestFactory().options(url, data, **kwargs) + if middleware: + middleware.process_request(request) + if user: + force_authenticate(request, user=user) + response = view(request, *view_args, **view_kwargs) + if middleware: + middleware.process_response(request, response) + return response + return rf + +@pytest.fixture +def instance(settings): + return Instance.objects.create(uuid=settings.SYSTEM_UUID, primary=True, hostname="instance.example.org") + +@pytest.fixture +def organization(instance): + return Organization.objects.create(name="test-org", description="test-org-desc") + +@pytest.fixture +@mock.patch.object(Project, "update", lambda self, **kwargs: None) +def project(instance): + return Project.objects.create(name="test-proj", + description="test-proj-desc", + scm_type="git", + scm_url="https://github.com/jlaska/ansible-playbooks") diff --git a/awx/main/tests/functional/test_notifications.py b/awx/main/tests/functional/test_notifications.py new file mode 100644 index 0000000000..72ac91b7b1 --- /dev/null +++ b/awx/main/tests/functional/test_notifications.py @@ -0,0 +1,124 @@ +import mock +import pytest + +from awx.main.models.notifications import Notification, Notifier +from awx.main.models.inventory import Inventory, Group +from awx.main.models.organization import Organization +from awx.main.models.projects import Project +from awx.main.models.jobs import JobTemplate + +from django.core.urlresolvers import reverse +from django.core.mail.message import EmailMessage + +@pytest.fixture +def notifier(): + return Notifier.objects.create(name="test-notification", + notification_type="webhook", + notification_configuration=dict(url="http://localhost", + headers={"Test": "Header"})) + +@pytest.mark.django_db +def test_get_notifier_list(get, user, notifier): + url = reverse('api:notifier_list') + response = get(url, user('admin', True)) + assert response.status_code == 200 + assert len(response.data['results']) == 1 + +@pytest.mark.django_db +def test_basic_parameterization(get, post, user, organization): + u = user('admin-poster', True) + url = reverse('api:notifier_list') + response = post(url, + dict(name="test-webhook", + description="test webhook", + organization=1, + notification_type="webhook", + notification_configuration=dict(url="http://localhost", + headers={"Test": "Header"})), + u) + assert response.status_code == 201 + url = reverse('api:notifier_detail', args=(response.data['id'],)) + response = get(url, u) + assert 'related' in response.data + assert 'organization' in response.data['related'] + assert 'summary_fields' in response.data + assert 'organization' in response.data['summary_fields'] + assert 'notifications' in response.data['related'] + assert 'notification_configuration' in response.data + assert 'url' in response.data['notification_configuration'] + assert 'headers' in response.data['notification_configuration'] + +@pytest.mark.django_db +def test_encrypted_subfields(get, post, user, organization): + def assert_send(self, messages): + assert self.account_token == "shouldhide" + return 1 + u = user('admin-poster', True) + url = reverse('api:notifier_list') + response = post(url, + dict(name="test-twilio", + description="test twilio", + organization=1, + notification_type="twilio", + notification_configuration=dict(account_sid="dummy", + account_token="shouldhide", + from_number="+19999999999", + to_numbers=["9998887777"])), + u) + assert response.status_code == 201 + notifier_actual = Notifier.objects.get(id=response.data['id']) + assert notifier_actual.notification_configuration['account_token'].startswith("$encrypted$") + url = reverse('api:notifier_detail', args=(response.data['id'],)) + response = get(url, u) + assert response.data['notification_configuration']['account_token'] == "$encrypted$" + with mock.patch.object(notifier_actual.notification_class, "send_messages", assert_send): + notifier_actual.send("Test", {'body': "Test"}) + +@pytest.mark.django_db +def test_inherited_notifiers(get, post, user, organization, project): + u = user('admin-poster', True) + url = reverse('api:notifier_list') + notifiers = [] + for nfiers in xrange(3): + response = post(url, + dict(name="test-webhook-{}".format(nfiers), + description="test webhook {}".format(nfiers), + organization=1, + notification_type="webhook", + notification_configuration=dict(url="http://localhost", + headers={"Test": "Header"})), + u) + assert response.status_code == 201 + notifiers.append(response.data['id']) + o = Organization.objects.get(id=1) + p = Project.objects.get(id=1) + o.projects.add(p) + i = Inventory.objects.create(name='test', organization=o) + i.save() + g = Group.objects.create(name='test', inventory=i) + g.save() + jt = JobTemplate.objects.create(name='test', inventory=i, project=p, playbook='debug.yml') + jt.save() + url = reverse('api:organization_notifiers_any_list', args=(1,)) + response = post(url, dict(id=notifiers[0]), u) + assert response.status_code == 204 + url = reverse('api:project_notifiers_any_list', args=(1,)) + response = post(url, dict(id=notifiers[1]), u) + assert response.status_code == 204 + url = reverse('api:job_template_notifiers_any_list', args=(jt.id,)) + response = post(url, dict(id=notifiers[2]), u) + assert response.status_code == 204 + assert len(jt.notifiers['any']) == 3 + assert len(p.notifiers['any']) == 2 + assert len(g.inventory_source.notifiers['any']) == 1 + +@pytest.mark.django_db +def test_notifier_merging(get, post, user, organization, project, notifier): + u = user('admin-poster', True) + o = Organization.objects.get(id=1) + p = Project.objects.get(id=1) + n = Notifier.objects.get(id=1) + o.projects.add(p) + o.notifiers_any.add(n) + p.notifiers_any.add(n) + assert len(p.notifiers['any']) == 1 From b892ee5f96dceb648aa64b075341752a1a551b9b Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Mon, 29 Feb 2016 13:25:49 -0500 Subject: [PATCH 23/23] Updates to notification unit tests after @wwitzel3's feedback --- .../tests/functional/test_notifications.py | 28 ++++++++----------- 1 file changed, 11 insertions(+), 17 deletions(-) diff --git a/awx/main/tests/functional/test_notifications.py b/awx/main/tests/functional/test_notifications.py index 72ac91b7b1..89beb5bcc6 100644 --- a/awx/main/tests/functional/test_notifications.py +++ b/awx/main/tests/functional/test_notifications.py @@ -58,7 +58,7 @@ def test_encrypted_subfields(get, post, user, organization): response = post(url, dict(name="test-twilio", description="test twilio", - organization=1, + organization=organization.id, notification_type="twilio", notification_configuration=dict(account_sid="dummy", account_token="shouldhide", @@ -67,7 +67,6 @@ def test_encrypted_subfields(get, post, user, organization): u) assert response.status_code == 201 notifier_actual = Notifier.objects.get(id=response.data['id']) - assert notifier_actual.notification_configuration['account_token'].startswith("$encrypted$") url = reverse('api:notifier_detail', args=(response.data['id'],)) response = get(url, u) assert response.data['notification_configuration']['account_token'] == "$encrypted$" @@ -90,35 +89,30 @@ def test_inherited_notifiers(get, post, user, organization, project): u) assert response.status_code == 201 notifiers.append(response.data['id']) - o = Organization.objects.get(id=1) - p = Project.objects.get(id=1) - o.projects.add(p) - i = Inventory.objects.create(name='test', organization=o) + organization.projects.add(project) + i = Inventory.objects.create(name='test', organization=organization) i.save() g = Group.objects.create(name='test', inventory=i) g.save() - jt = JobTemplate.objects.create(name='test', inventory=i, project=p, playbook='debug.yml') + jt = JobTemplate.objects.create(name='test', inventory=i, project=project, playbook='debug.yml') jt.save() - url = reverse('api:organization_notifiers_any_list', args=(1,)) + url = reverse('api:organization_notifiers_any_list', args=(organization.id,)) response = post(url, dict(id=notifiers[0]), u) assert response.status_code == 204 - url = reverse('api:project_notifiers_any_list', args=(1,)) + url = reverse('api:project_notifiers_any_list', args=(project.id,)) response = post(url, dict(id=notifiers[1]), u) assert response.status_code == 204 url = reverse('api:job_template_notifiers_any_list', args=(jt.id,)) response = post(url, dict(id=notifiers[2]), u) assert response.status_code == 204 assert len(jt.notifiers['any']) == 3 - assert len(p.notifiers['any']) == 2 + assert len(project.notifiers['any']) == 2 assert len(g.inventory_source.notifiers['any']) == 1 @pytest.mark.django_db def test_notifier_merging(get, post, user, organization, project, notifier): u = user('admin-poster', True) - o = Organization.objects.get(id=1) - p = Project.objects.get(id=1) - n = Notifier.objects.get(id=1) - o.projects.add(p) - o.notifiers_any.add(n) - p.notifiers_any.add(n) - assert len(p.notifiers['any']) == 1 + organization.projects.add(project) + organization.notifiers_any.add(notifier) + project.notifiers_any.add(notifier) + assert len(project.notifiers['any']) == 1