Add AWS SNS notification support for webhook (#15184)

Support for AWS SNS notifications. SNS is a widespread service that is used to integrate with other AWS services(EG lambdas). This support would unlock use cases like triggering lambda functions, especially when AWX is deployed on EKS.

Decisions:

Data Structure
- I preferred using the same structure as Webhook for message body data because it contains all job details. For now, I directly linked to Webhook to avoid duplication, but I am open to suggestions.

AWS authentication
- To support non-AWS native environments, I added configuration options for AWS secret key, ID, and session tokens. When entered, these values are supplied to the underlining boto3 SNS client. If not entered, it falls back to the default authentication chain to support the native AWS environment. Properly configured EKS pods are created with temporary credentials that the default authentication chain can pick automatically.

---------

Signed-off-by: Ethem Cem Ozkan <ethemcem.ozkan@gmail.com>

awx/api/serializers.py

@@ -5381,7 +5381,7 @@ class NotificationSerializer(BaseSerializer):
         )
 
     def get_body(self, obj):
-        if obj.notification_type in ('webhook', 'pagerduty'):
+        if obj.notification_type in ('webhook', 'pagerduty', 'awssns'):
             if isinstance(obj.body, dict):
                 if 'body' in obj.body:
                     return obj.body['body']
@@ -5403,9 +5403,9 @@ class NotificationSerializer(BaseSerializer):
     def to_representation(self, obj):
         ret = super(NotificationSerializer, self).to_representation(obj)
 
-        if obj.notification_type == 'webhook':
+        if obj.notification_type in ('webhook', 'awssns'):
             ret.pop('subject')
-        if obj.notification_type not in ('email', 'webhook', 'pagerduty'):
+        if obj.notification_type not in ('email', 'webhook', 'pagerduty', 'awssns'):
             ret.pop('body')
         return ret
 

awx/main/migrations/0193_alter_notification_notification_type_and_more.py

@@ -0,0 +1,51 @@
+# Generated by Django 4.2.6 on 2024-05-08 07:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0192_custom_roles'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='notification',
+            name='notification_type',
+            field=models.CharField(
+                choices=[
+                    ('awssns', 'AWS SNS'),
+                    ('email', 'Email'),
+                    ('grafana', 'Grafana'),
+                    ('irc', 'IRC'),
+                    ('mattermost', 'Mattermost'),
+                    ('pagerduty', 'Pagerduty'),
+                    ('rocketchat', 'Rocket.Chat'),
+                    ('slack', 'Slack'),
+                    ('twilio', 'Twilio'),
+                    ('webhook', 'Webhook'),
+                ],
+                max_length=32,
+            ),
+        ),
+        migrations.AlterField(
+            model_name='notificationtemplate',
+            name='notification_type',
+            field=models.CharField(
+                choices=[
+                    ('awssns', 'AWS SNS'),
+                    ('email', 'Email'),
+                    ('grafana', 'Grafana'),
+                    ('irc', 'IRC'),
+                    ('mattermost', 'Mattermost'),
+                    ('pagerduty', 'Pagerduty'),
+                    ('rocketchat', 'Rocket.Chat'),
+                    ('slack', 'Slack'),
+                    ('twilio', 'Twilio'),
+                    ('webhook', 'Webhook'),
+                ],
+                max_length=32,
+            ),
+        ),
+    ]

awx/main/models/notifications.py

@@ -31,6 +31,7 @@ from awx.main.notifications.mattermost_backend import MattermostBackend
 from awx.main.notifications.grafana_backend import GrafanaBackend
 from awx.main.notifications.rocketchat_backend import RocketChatBackend
 from awx.main.notifications.irc_backend import IrcBackend
+from awx.main.notifications.awssns_backend import AWSSNSBackend
 
 
 logger = logging.getLogger('awx.main.models.notifications')
@@ -40,6 +41,7 @@ __all__ = ['NotificationTemplate', 'Notification']
 
 class NotificationTemplate(CommonModelNameNotUnique):
     NOTIFICATION_TYPES = [
+        ('awssns', _('AWS SNS'), AWSSNSBackend),
         ('email', _('Email'), CustomEmailBackend),
         ('slack', _('Slack'), SlackBackend),
         ('twilio', _('Twilio'), TwilioBackend),

awx/main/notifications/awssns_backend.py

@@ -0,0 +1,70 @@
+# Copyright (c) 2016 Ansible, Inc.
+# All Rights Reserved.
+import json
+import logging
+
+import boto3
+from botocore.exceptions import ClientError
+
+from awx.main.notifications.base import AWXBaseEmailBackend
+from awx.main.notifications.custom_notification_base import CustomNotificationBase
+
+logger = logging.getLogger('awx.main.notifications.awssns_backend')
+WEBSOCKET_TIMEOUT = 30
+
+
+class AWSSNSBackend(AWXBaseEmailBackend, CustomNotificationBase):
+    init_parameters = {
+        "aws_region": {"label": "AWS Region", "type": "string", "default": ""},
+        "aws_access_key_id": {"label": "Access Key ID", "type": "string", "default": ""},
+        "aws_secret_access_key": {"label": "Secret Access Key", "type": "password", "default": ""},
+        "aws_session_token": {"label": "Session Token", "type": "password", "default": ""},
+        "sns_topic_arn": {"label": "SNS Topic ARN", "type": "string", "default": ""},
+    }
+    recipient_parameter = "sns_topic_arn"
+    sender_parameter = None
+
+    DEFAULT_BODY = "{{ job_metadata }}"
+    default_messages = CustomNotificationBase.job_metadata_messages
+
+    def __init__(self, aws_region, aws_access_key_id, aws_secret_access_key, aws_session_token, fail_silently=False, **kwargs):
+        session = boto3.session.Session()
+        client_config = {"service_name": 'sns'}
+        if aws_region:
+            client_config["region_name"] = aws_region
+        if aws_secret_access_key:
+            client_config["aws_secret_access_key"] = aws_secret_access_key
+        if aws_access_key_id:
+            client_config["aws_access_key_id"] = aws_access_key_id
+        if aws_session_token:
+            client_config["aws_session_token"] = aws_session_token
+        self.client = session.client(**client_config)
+        super(AWSSNSBackend, self).__init__(fail_silently=fail_silently)
+
+    def _sns_publish(self, topic_arn, message):
+        self.client.publish(TopicArn=topic_arn, Message=message, MessageAttributes={})
+
+    def format_body(self, body):
+        if isinstance(body, str):
+            try:
+                body = json.loads(body)
+            except json.JSONDecodeError:
+                pass
+
+        if isinstance(body, dict):
+            body = json.dumps(body)
+        # convert dict body to json string
+        return body
+
+    def send_messages(self, messages):
+        sent_messages = 0
+        for message in messages:
+            sns_topic_arn = str(message.recipients()[0])
+            try:
+                self._sns_publish(topic_arn=sns_topic_arn, message=message.body)
+                sent_messages += 1
+            except ClientError as error:
+                if not self.fail_silently:
+                    raise error
+
+        return sent_messages

awx/main/notifications/custom_notification_base.py

@@ -32,3 +32,15 @@ class CustomNotificationBase(object):
             "denied": {"message": DEFAULT_APPROVAL_DENIED_MSG, "body": None},
         },
     }
+
+    job_metadata_messages = {
+        "started": {"body": "{{ job_metadata }}"},
+        "success": {"body": "{{ job_metadata }}"},
+        "error": {"body": "{{ job_metadata }}"},
+        "workflow_approval": {
+            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. This node can be viewed at: {{ workflow_url }}"}'},
+            "approved": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was approved. {{ workflow_url }}"}'},
+            "timed_out": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" has timed out. {{ workflow_url }}"}'},
+            "denied": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was denied. {{ workflow_url }}"}'},
+        },
+    }

awx/main/notifications/webhook_backend.py

@@ -27,17 +27,7 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
     sender_parameter = None
 
     DEFAULT_BODY = "{{ job_metadata }}"
-    default_messages = {
-        "started": {"body": DEFAULT_BODY},
-        "success": {"body": DEFAULT_BODY},
-        "error": {"body": DEFAULT_BODY},
-        "workflow_approval": {
-            "running": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" needs review. This node can be viewed at: {{ workflow_url }}"}'},
-            "approved": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was approved. {{ workflow_url }}"}'},
-            "timed_out": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" has timed out. {{ workflow_url }}"}'},
-            "denied": {"body": '{"body": "The approval node \\"{{ approval_node_name }}\\" was denied. {{ workflow_url }}"}'},
-        },
-    }
+    default_messages = CustomNotificationBase.job_metadata_messages
 
     def __init__(self, http_method, headers, disable_ssl_verification=False, fail_silently=False, username=None, password=None, **kwargs):
         self.http_method = http_method

awx/main/tests/unit/notifications/test_awssns.py

@@ -0,0 +1,26 @@
+from unittest import mock
+from django.core.mail.message import EmailMessage
+
+import awx.main.notifications.awssns_backend as awssns_backend
+
+
+def test_send_messages():
+    with mock.patch('awx.main.notifications.awssns_backend.AWSSNSBackend._sns_publish') as sns_publish_mock:
+        aws_region = 'us-east-1'
+        sns_topic = f"arn:aws:sns:{aws_region}:111111111111:topic-mock"
+        backend = awssns_backend.AWSSNSBackend(aws_region=aws_region, aws_access_key_id=None, aws_secret_access_key=None, aws_session_token=None)
+        message = EmailMessage(
+            'test subject',
+            {'body': 'test body'},
+            [],
+            [
+                sns_topic,
+            ],
+        )
+        sent_messages = backend.send_messages(
+            [
+                message,
+            ]
+        )
+        sns_publish_mock.assert_called_once_with(topic_arn=sns_topic, message=message.body)
+        assert sent_messages == 1

awx/ui/src/components/NotificationList/NotificationList.js

@@ -190,6 +190,7 @@ function NotificationList({
             name: t`Notification type`,
             key: 'or__notification_type',
             options: [
+              ['awssns', t`AWS SNS`],
               ['email', t`Email`],
               ['grafana', t`Grafana`],
               ['hipchat', t`Hipchat`],

awx/ui/src/screens/NotificationTemplate/NotificationTemplateDetail/NotificationTemplateDetail.js

@@ -138,6 +138,25 @@ function NotificationTemplateDetail({ template, defaultMessages }) {
           }
           dataCy="nt-detail-type"
         />
+        {template.notification_type === 'awssns' && (
+          <>
+            <Detail
+              label={t`AWS Region`}
+              value={configuration.aws_region}
+              dataCy="nt-detail-aws-region"
+            />
+            <Detail
+              label={t`Access Key ID`}
+              value={configuration.aws_access_key_id}
+              dataCy="nt-detail-aws-access-key-id"
+            />
+            <Detail
+              label={t`SNS Topic ARN`}
+              value={configuration.sns_topic_arn}
+              dataCy="nt-detail-sns-topic-arn"
+            />
+          </>
+        )}
         {template.notification_type === 'email' && (
           <>
             <Detail
@@ -455,8 +474,8 @@ function NotificationTemplateDetail({ template, defaultMessages }) {
 }
 
 function CustomMessageDetails({ messages, defaults, type }) {
-  const showMessages = type !== 'webhook';
-  const showBodies = ['email', 'pagerduty', 'webhook'].includes(type);
+  const showMessages = !['awssns', 'webhook'].includes(type);
+  const showBodies = ['email', 'pagerduty', 'webhook', 'awssns'].includes(type);
 
   return (
     <>

awx/ui/src/screens/NotificationTemplate/NotificationTemplateList/NotificationTemplateList.js

@@ -131,6 +131,7 @@ function NotificationTemplatesList() {
                 name: t`Notification type`,
                 key: 'or__notification_type',
                 options: [
+                  ['awssns', t`AWS SNS`],
                   ['email', t`Email`],
                   ['grafana', t`Grafana`],
                   ['hipchat', t`Hipchat`],

awx/ui/src/screens/NotificationTemplate/constants.js

@@ -1,5 +1,6 @@
 /* eslint-disable-next-line import/prefer-default-export */
 export const NOTIFICATION_TYPES = {
+  awssns: 'AWS SNS',
   email: 'Email',
   grafana: 'Grafana',
   irc: 'IRC',

awx/ui/src/screens/NotificationTemplate/shared/CustomMessagesSubForm.js

@@ -11,8 +11,8 @@ import getDocsBaseUrl from 'util/getDocsBaseUrl';
 
 function CustomMessagesSubForm({ defaultMessages, type }) {
   const [useCustomField, , useCustomHelpers] = useField('useCustomMessages');
-  const showMessages = type !== 'webhook';
-  const showBodies = ['email', 'pagerduty', 'webhook'].includes(type);
+  const showMessages = !['webhook', 'awssns'].includes(type);
+  const showBodies = ['email', 'pagerduty', 'webhook', 'awssns'].includes(type);
 
   const { setFieldValue } = useFormikContext();
   const config = useConfig();

awx/ui/src/screens/NotificationTemplate/shared/NotificationTemplateForm.js

@@ -78,6 +78,7 @@ function NotificationTemplateFormFields({ defaultMessages, template }) {
               label: t`Choose a Notification Type`,
               isDisabled: true,
             },
+            { value: 'awssns', key: 'awssns', label: t`AWS SNS` },
             { value: 'email', key: 'email', label: t`E-mail` },
             { value: 'grafana', key: 'grafana', label: 'Grafana' },
             { value: 'irc', key: 'irc', label: 'IRC' },

awx/ui/src/screens/NotificationTemplate/shared/TypeInputsSubForm.js

@@ -29,6 +29,7 @@ import Popover from '../../../components/Popover/Popover';
 import getHelpText from './Notifications.helptext';
 
 const TypeFields = {
+  awssns: AWSSNSFields,
   email: EmailFields,
   grafana: GrafanaFields,
   irc: IRCFields,
@@ -58,6 +59,44 @@ TypeInputsSubForm.propTypes = {
 
 export default TypeInputsSubForm;
 
+function AWSSNSFields() {
+  return (
+    <>
+      <FormField
+        id="awssns-aws-region"
+        label={t`AWS Region`}
+        name="notification_configuration.aws_region"
+        type="text"
+        isRequired
+      />
+      <FormField
+        id="awssns-aws-access-key-id"
+        label={t`Access Key ID`}
+        name="notification_configuration.aws_access_key_id"
+        type="text"
+      />
+      <PasswordField
+        id="awssns-aws-secret-access-key"
+        label={t`Secret Access Key`}
+        name="notification_configuration.aws_secret_access_key"
+      />
+      <PasswordField
+        id="awssns-aws-session-token"
+        label={t`Session Token`}
+        name="notification_configuration.aws_session_token"
+      />
+      <FormField
+        id="awssns-sns-topic-arn"
+        label={t`SNS Topic ARN`}
+        name="notification_configuration.sns_topic_arn"
+        type="text"
+        validate={required(null)}
+        isRequired
+      />
+    </>
+  );
+}
+
 function EmailFields() {
   const helpText = getHelpText();
   return (

awx/ui/src/screens/NotificationTemplate/shared/notification-template-default-messages.json

@@ -203,6 +203,39 @@
       }
     }
   },
+  "awssns": {
+    "started": {
+      "body": "{{ job_metadata }}"
+    },
+    "success": {
+      "body": "{{ job_metadata }}"
+    },
+    "error": {
+      "body": "{{ job_metadata }}"
+    },
+    "workflow_approval": {
+      "running": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" needs review. This node can be viewed at: {{ workflow_url }}"
+        }
+      },
+      "approved": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" was approved. {{ workflow_url }}"
+        }
+      },
+      "timed_out": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" has timed out. {{ workflow_url }}"
+        }
+      },
+      "denied": {
+        "body": {
+          "body": "The approval node \"{{ approval_node_name }}\" was denied. {{ workflow_url }}"
+        }
+      }
+    }
+  },
   "mattermost": {
     "started": {
       "message": "{{ job_friendly_name }} #{{ job.id }} '{{ job.name }}' {{ job.status }}: {{ url }}",

awx/ui/src/screens/NotificationTemplate/shared/typeFieldNames.js

@@ -1,4 +1,11 @@
 const typeFieldNames = {
+  awssns: [
+    'aws_region',
+    'aws_access_key_id',
+    'aws_secret_access_key',
+    'aws_session_token',
+    'sns_topic_arn',
+  ],
   email: [
     'username',
     'password',

awx/ui/src/types.js

@@ -374,6 +374,7 @@ export const CredentialType = shape({
 });
 
 export const NotificationType = oneOf([
+  'awssns',
   'email',
   'grafana',
   'irc',

awx_collection/plugins/modules/notification_template.py

@@ -50,6 +50,7 @@ options:
       description:
         - The type of notification to be sent.
       choices:
+        - 'awssns'
         - 'email'
         - 'grafana'
         - 'irc'
@@ -219,7 +220,7 @@ def main():
         copy_from=dict(),
         description=dict(),
         organization=dict(),
-        notification_type=dict(choices=['email', 'grafana', 'irc', 'mattermost', 'pagerduty', 'rocketchat', 'slack', 'twilio', 'webhook']),
+        notification_type=dict(choices=['awssns', 'email', 'grafana', 'irc', 'mattermost', 'pagerduty', 'rocketchat', 'slack', 'twilio', 'webhook']),
         notification_configuration=dict(type='dict'),
         messages=dict(type='dict'),
         state=dict(choices=['present', 'absent', 'exists'], default='present'),

awxkit/awxkit/api/pages/notification_templates.py

@@ -11,7 +11,7 @@ from . import page
 
 
 job_results = ('any', 'error', 'success')
-notification_types = ('email', 'irc', 'pagerduty', 'slack', 'twilio', 'webhook', 'mattermost', 'grafana', 'rocketchat')
+notification_types = ('awssns', 'email', 'irc', 'pagerduty', 'slack', 'twilio', 'webhook', 'mattermost', 'grafana', 'rocketchat')
 
 
 class NotificationTemplate(HasCopy, HasCreate, base.Base):
@@ -58,7 +58,10 @@ class NotificationTemplate(HasCopy, HasCreate, base.Base):
         if payload.notification_configuration == {}:
             services = config.credentials.notification_services
 
-            if notification_type == 'email':
+            if notification_type == 'awssns':
+                fields = ('aws_region', 'aws_access_key_id', 'aws_secret_access_key', 'aws_session_token', 'sns_topic_arn')
+                cred = services.awssns
+            elif notification_type == 'email':
                 fields = ('host', 'username', 'password', 'port', 'use_ssl', 'use_tls', 'sender', 'recipients')
                 cred = services.email
             elif notification_type == 'irc':

docs/docsite/rst/userguide/notifications.rst

@@ -84,6 +84,7 @@ Notification Types
 
 .. index:: 
    pair: notifications; types
+   triple: notifications; types; AWS SNS
    triple: notifications; types; Email
    triple: notifications; types; Grafana
    triple: notifications; types; IRC
@@ -101,6 +102,18 @@ Notification types supported with AWX:
 
 Each of these have their own configuration and behavioral semantics and testing them may need to be approached in different ways. Additionally, you can customize each type of notification down to a specific detail, or a set of criteria to trigger a notification. See :ref:`ug_custom_notifications` for more detail on configuring custom notifications. The following sections will give as much detail as possible on each type of notification.
 
+AWS SNS
+-------
+
+The AWS SNS(https://aws.amazon.com/sns/) notification type supports sending messages into an SNS topic.
+
+You must provide the following details to setup a SNS notification:
+
+- AWS Region
+- AWS Access Key ID
+- AWS Secret Access Key
+- AWS SNS Topic ARN
+
 
 Email
 -------

docs/notification_system.md

@@ -70,6 +70,7 @@ Once a Notification Template has been created, its configuration can be tested b
 
 The currently-defined Notification Types are:
 
+* AWS SNS
 * Email
 * Slack
 * Mattermost
@@ -82,6 +83,10 @@ The currently-defined Notification Types are:
 
 Each of these have their own configuration and behavioral semantics and testing them may need to be approached in different ways. The following sections will give as much detail as possible.
 
+## AWS SNS
+
+The AWS SNS notification type supports sending messages into an SNS topic.
+
 ## Email
 
 The email notification type supports a wide variety of SMTP servers and has support for SSL/TLS connections and timeouts.