From fb53ca8455829b78b394ee3ba9898f82e20586d4 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 18 May 2017 17:22:29 -0400 Subject: [PATCH 01/10] insights proxy * Issue request to Red Hat Insights API from Tower /hosts//insights endpoint. User the first found Satellite 6 Credential as Basic Auth requests to Insights API. --- awx/api/serializers.py | 1 + awx/api/urls.py | 1 + awx/api/views.py | 42 +++++++++++++++++++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index f5a27baefc..7045f5c037 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1205,6 +1205,7 @@ class HostSerializer(BaseSerializerWithVariables): ad_hoc_commands = self.reverse('api:host_ad_hoc_commands_list', kwargs={'pk': obj.pk}), ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}), fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}), + insights = self.reverse('api:host_insights', kwargs={'pk': obj.pk}), )) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) diff --git a/awx/api/urls.py b/awx/api/urls.py index 80f5eb7248..f764ff3ffd 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -120,6 +120,7 @@ host_urls = patterns('awx.api.views', #url(r'^(?P[0-9]+)/single_fact/$', 'host_single_fact_view'), url(r'^(?P[0-9]+)/fact_versions/$', 'host_fact_versions_list'), url(r'^(?P[0-9]+)/fact_view/$', 'host_fact_compare_view'), + url(r'^(?P[0-9]+)/insights/$', 'host_insights'), ) group_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index 8116f35bbb..2e3c25da0b 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -13,6 +13,7 @@ import socket import subprocess import sys import logging +import requests from base64 import b64encode from collections import OrderedDict @@ -70,7 +71,8 @@ from awx.conf.license import get_license, feature_enabled, feature_exists, Licen from awx.main.models import * # noqa from awx.main.utils import * # noqa from awx.main.utils import ( - callback_filter_out_ansible_extra_vars + callback_filter_out_ansible_extra_vars, + decrypt_field, ) from awx.main.utils.filters import SmartFilter @@ -2067,6 +2069,44 @@ class HostFactCompareView(SystemTrackingEnforcementMixin, SubDetailAPIView): return Response(self.serializer_class(instance=fact_entry).data) +class HostInsights(GenericAPIView): + + model = Host + serializer_class = EmptySerializer + + def get(self, request, *args, **kwargs): + host = self.get_object() + cred = None + + if host.insights_system_id is None: + return Response(status=status.HTTP_204_NO_CONTENT) + + creds = Credential.objects.filter(credential_type__name='Red Hat Satellite 6', credential_type__kind='cloud', credential_type__managed_by_tower=True) + if creds.count() > 0: + cred = creds[0] + else: + ''' + TODO: Different ERROR code? .. definately add more information feedback in 'errors' key + ''' + return Response(status=status.HTTP_204_NO_CONTENT) + + username = cred.inputs['username'] + password = decrypt_field(cred, 'password') + + session = requests.Session() + session.auth = requests.auth.HTTPBasicAuth(username, password) + headers = {'Content-Type': 'application/json'} + res = session.get('https://access.redhat.com/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id), headers=headers) + + if res.status_code != 200: + return Response(status=status.HTTP_204_NO_CONTENT) + + try: + return Response(res.json()) + except ValueError: + return Response(status=status.HTTP_204_NO_CONTENT) + + class GroupList(ListCreateAPIView): model = Group From 9ba0ba07ec5fb79ea5bc315714ca3b2bd547ecfd Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 19 May 2017 16:37:30 -0400 Subject: [PATCH 02/10] nest insights api response data --- awx/api/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/awx/api/views.py b/awx/api/views.py index 2e3c25da0b..45933b4bda 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2077,6 +2077,7 @@ class HostInsights(GenericAPIView): def get(self, request, *args, **kwargs): host = self.get_object() cred = None + ret = {} if host.insights_system_id is None: return Response(status=status.HTTP_204_NO_CONTENT) @@ -2102,7 +2103,8 @@ class HostInsights(GenericAPIView): return Response(status=status.HTTP_204_NO_CONTENT) try: - return Response(res.json()) + ret['insights_content'] = res.json() + return Response(ret) except ValueError: return Response(status=status.HTTP_204_NO_CONTENT) From 3793f1ec0914ed1ca3f8b8f386f97256cbc8b125 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 22 May 2017 15:02:16 -0400 Subject: [PATCH 03/10] insights proxy robust error condition handling --- awx/api/views.py | 53 ++++++++++++++++++++++++++++-------------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 45933b4bda..76ca0a0130 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2074,39 +2074,48 @@ class HostInsights(GenericAPIView): model = Host serializer_class = EmptySerializer + def _extract_insights_creds(self, credential): + return (credential.inputs['username'], decrypt_field(credential, 'password')) + + def _get_insights(self, url, username, password): + session = requests.Session() + session.auth = requests.auth.HTTPBasicAuth(username, password) + headers = {'Content-Type': 'application/json'} + return session.get(url, headers=headers, timeout=120) + + def get_insights(self, url, username, password): + try: + res = self._get_insights(url, username, password) + except requests.exceptions.SSLError: + return (dict(error='SSLError while trying to connect to https://access.redhat.com/'), status.HTTP_500_INTERNAL_SERVER_ERROR) + except requests.exceptions.Timeout: + return (dict(error='Request to {} timed out'.format(url)), status.HTTP_504_GATEWAY_TIMEOUT) + + if res.status_code != 200: + return (dict(error='Failed to gather reports and maintenance plans from Insights API. Server responded with {} status code and message {}'.format(res.status_code, res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) + + try: + return (dict(insights_content=res.json()), status.HTTP_200_OK) + except ValueError: + return (None, status.HTTP_204_NO_CONTENT) + def get(self, request, *args, **kwargs): host = self.get_object() cred = None - ret = {} if host.insights_system_id is None: - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(dict(error='This host is not recognized as an Insights host.'), status=status.HTTP_404_NOT_FOUND) creds = Credential.objects.filter(credential_type__name='Red Hat Satellite 6', credential_type__kind='cloud', credential_type__managed_by_tower=True) if creds.count() > 0: cred = creds[0] else: - ''' - TODO: Different ERROR code? .. definately add more information feedback in 'errors' key - ''' - return Response(status=status.HTTP_204_NO_CONTENT) + return Response(dict(error='No Insights Credential found for the Inventory, "{}", that this host belongs to.'.format(host.inventory.name)), status=status.HTTP_404_NOT_FOUND) - username = cred.inputs['username'] - password = decrypt_field(cred, 'password') - - session = requests.Session() - session.auth = requests.auth.HTTPBasicAuth(username, password) - headers = {'Content-Type': 'application/json'} - res = session.get('https://access.redhat.com/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id), headers=headers) - - if res.status_code != 200: - return Response(status=status.HTTP_204_NO_CONTENT) - - try: - ret['insights_content'] = res.json() - return Response(ret) - except ValueError: - return Response(status=status.HTTP_204_NO_CONTENT) + url = 'https://access.redhat.com/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id) + (username, password) = self._extract_insights_creds(cred) + (msg, status) = self.get_insights(url, username, password) + return Response(msg, status=status) class GroupList(ListCreateAPIView): From 4461c8fe91ac2816d594491336575b7e4425e2b9 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 22 May 2017 17:32:17 -0400 Subject: [PATCH 04/10] more robust insights error handling --- awx/api/views.py | 9 +++-- awx/main/tests/unit/api/test_views.py | 47 +++++++++++++++++++++++++++ 2 files changed, 53 insertions(+), 3 deletions(-) diff --git a/awx/api/views.py b/awx/api/views.py index 76ca0a0130..b61db39437 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -14,6 +14,7 @@ import subprocess import sys import logging import requests +import urlparse from base64 import b64encode from collections import OrderedDict @@ -2087,9 +2088,11 @@ class HostInsights(GenericAPIView): try: res = self._get_insights(url, username, password) except requests.exceptions.SSLError: - return (dict(error='SSLError while trying to connect to https://access.redhat.com/'), status.HTTP_500_INTERNAL_SERVER_ERROR) + return (dict(error='SSLError while trying to connect to {}'.format(url)), status.HTTP_500_INTERNAL_SERVER_ERROR) except requests.exceptions.Timeout: - return (dict(error='Request to {} timed out'.format(url)), status.HTTP_504_GATEWAY_TIMEOUT) + return (dict(error='Request to {} timed out.'.format(url)), status.HTTP_504_GATEWAY_TIMEOUT) + except requests.exceptions.RequestException as e: + return (dict(error='Unkown exception {} while trying to GET {}'.format(e, url)), status.HTTP_500_INTERNAL_SERVER_ERROR) if res.status_code != 200: return (dict(error='Failed to gather reports and maintenance plans from Insights API. Server responded with {} status code and message {}'.format(res.status_code, res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) @@ -2097,7 +2100,7 @@ class HostInsights(GenericAPIView): try: return (dict(insights_content=res.json()), status.HTTP_200_OK) except ValueError: - return (None, status.HTTP_204_NO_CONTENT) + return (dict(error='Expected JSON response from Insights but instead got {}'.format(res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) def get(self, request, *args, **kwargs): host = self.get_object() diff --git a/awx/main/tests/unit/api/test_views.py b/awx/main/tests/unit/api/test_views.py index e4291e50cb..837d202044 100644 --- a/awx/main/tests/unit/api/test_views.py +++ b/awx/main/tests/unit/api/test_views.py @@ -1,5 +1,6 @@ import mock import pytest +import requests from collections import namedtuple @@ -8,6 +9,7 @@ from awx.api.views import ( JobTemplateLabelList, JobTemplateSurveySpec, InventoryInventorySourcesUpdate, + HostInsights, ) @@ -117,3 +119,48 @@ class TestInventoryInventorySourcesUpdate: view = InventoryInventorySourcesUpdate() response = view.post(mock_request) assert response.data == expected + + +class TestHostInsights(): + + @pytest.fixture + def patch_parent(self, mocker): + mocker.patch('awx.api.generics.GenericAPIView') + + @pytest.mark.parametrize("status_code, exception, error, message", [ + (500, requests.exceptions.SSLError, 'SSLError while trying to connect to https://myexample.com/whocares/me/', None,), + (504, requests.exceptions.Timeout, 'Request to https://myexample.com/whocares/me/ timed out.', None,), + (500, requests.exceptions.RequestException, 'booo!', 'Unkown exception booo! while trying to GET https://myexample.com/whocares/me/'), + ]) + def test_get_insights_request_exception(self, patch_parent, mocker, status_code, exception, error, message): + view = HostInsights() + mocker.patch.object(view, '_get_insights', side_effect=exception(error)) + + (msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore') + assert code == status_code + assert msg['error'] == message or error + + def test_get_insights_non_200(self, patch_parent, mocker): + view = HostInsights() + Response = namedtuple('Response', 'status_code content') + mocker.patch.object(view, '_get_insights', return_value=Response(500, 'hello world!')) + + (msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore') + assert msg['error'] == 'Failed to gather reports and maintenance plans from Insights API. Server responded with 500 status code and message hello world!' + + def test_get_insights_malformed_json_content(self, patch_parent, mocker): + view = HostInsights() + + class Response(): + status_code = 200 + content = 'booo!' + + def json(self): + raise ValueError('we do not care what this is') + + mocker.patch.object(view, '_get_insights', return_value=Response()) + + (msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore') + assert msg['error'] == 'Expected JSON response from Insights but instead got booo!' + assert code == 500 + From ca1eb28d1f859bb40baeba0a85d4c7f2b129a77c Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 25 May 2017 17:52:46 -0400 Subject: [PATCH 05/10] don't assume insights id is a uuid --- awx/plugins/library/scan_insights.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/awx/plugins/library/scan_insights.py b/awx/plugins/library/scan_insights.py index b7e625ca6d..2e759a28cb 100755 --- a/awx/plugins/library/scan_insights.py +++ b/awx/plugins/library/scan_insights.py @@ -1,14 +1,13 @@ #!/usr/bin/env python from ansible.module_utils.basic import * # noqa -import uuid DOCUMENTATION = ''' --- module: scan_insights -short_description: Return insights UUID as fact data +short_description: Return insights id as fact data description: - - Inspects the /etc/redhat-access-insights/machine-id file for insights uuid and returns the found UUID as fact data + - Inspects the /etc/redhat-access-insights/machine-id file for insights id and returns the found id as fact data version_added: "2.3" options: requirements: [ ] @@ -28,8 +27,8 @@ EXAMPLES = ''' INSIGHTS_SYSTEM_ID_FILE='/etc/redhat-access-insights/machine-id' -def get_system_uuid(filname): - system_uuid = None +def get_system_id(filname): + system_id = None try: f = open(INSIGHTS_SYSTEM_ID_FILE, "r") except IOError: @@ -37,12 +36,12 @@ def get_system_uuid(filname): else: try: data = f.readline() - system_uuid = str(uuid.UUID(data)) + system_id = str(data) except (IOError, ValueError): pass finally: f.close() - return system_uuid + return system_id def main(): @@ -50,12 +49,12 @@ def main(): argument_spec = dict() ) - system_uuid = get_system_uuid(INSIGHTS_SYSTEM_ID_FILE) + system_id = get_system_id(INSIGHTS_SYSTEM_ID_FILE) results = { 'ansible_facts': { 'insights': { - 'system_id': system_uuid + 'system_id': system_id } } } From 87eea59845d5b3357cda6298349849f6c99a376f Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Thu, 25 May 2017 17:53:51 -0400 Subject: [PATCH 06/10] add per-inventory insights credential --- awx/api/serializers.py | 5 ++- awx/api/views.py | 26 ++++++------ awx/main/migrations/0038_v320_release.py | 10 +++++ .../0040_v320_add_credentialtype_model.py | 2 +- awx/main/models/credential.py | 32 +++++++++++++- awx/main/models/inventory.py | 15 +++++++ awx/main/models/projects.py | 5 ++- .../tests/functional/api/test_inventory.py | 14 +++++++ awx/main/tests/functional/api/test_project.py | 15 +++++++ awx/main/tests/functional/conftest.py | 38 +++++++++++++++++ awx/main/tests/functional/test_credential.py | 1 + awx/main/tests/unit/api/test_views.py | 42 ++++++++++++++++++- 12 files changed, 185 insertions(+), 20 deletions(-) create mode 100644 awx/main/tests/functional/api/test_project.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 7045f5c037..57df11ef09 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1112,7 +1112,8 @@ class InventorySerializer(BaseSerializerWithVariables): fields = ('*', 'organization', 'kind', 'host_filter', 'variables', 'has_active_failures', 'total_hosts', 'hosts_with_active_failures', 'total_groups', 'groups_with_active_failures', 'has_inventory_sources', - 'total_inventory_sources', 'inventory_sources_with_failures') + 'total_inventory_sources', 'inventory_sources_with_failures', + 'insights_credential',) def get_related(self, obj): res = super(InventorySerializer, self).get_related(obj) @@ -1133,6 +1134,8 @@ class InventorySerializer(BaseSerializerWithVariables): object_roles = self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}), instance_groups = self.reverse('api:inventory_instance_groups_list', kwargs={'pk': obj.pk}), )) + if obj.insights_credential: + res['insights_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.insights_credential.pk}) if obj.organization: res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk}) return res diff --git a/awx/api/views.py b/awx/api/views.py index b61db39437..89b682bdbd 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -14,7 +14,6 @@ import subprocess import sys import logging import requests -import urlparse from base64 import b64encode from collections import OrderedDict @@ -2088,37 +2087,36 @@ class HostInsights(GenericAPIView): try: res = self._get_insights(url, username, password) except requests.exceptions.SSLError: - return (dict(error='SSLError while trying to connect to {}'.format(url)), status.HTTP_500_INTERNAL_SERVER_ERROR) + return (dict(error=_('SSLError while trying to connect to {}').format(url)), status.HTTP_500_INTERNAL_SERVER_ERROR) except requests.exceptions.Timeout: - return (dict(error='Request to {} timed out.'.format(url)), status.HTTP_504_GATEWAY_TIMEOUT) + return (dict(error=_('Request to {} timed out.').format(url)), status.HTTP_504_GATEWAY_TIMEOUT) except requests.exceptions.RequestException as e: - return (dict(error='Unkown exception {} while trying to GET {}'.format(e, url)), status.HTTP_500_INTERNAL_SERVER_ERROR) + return (dict(error=_('Unkown exception {} while trying to GET {}').format(e, url)), status.HTTP_500_INTERNAL_SERVER_ERROR) if res.status_code != 200: - return (dict(error='Failed to gather reports and maintenance plans from Insights API. Server responded with {} status code and message {}'.format(res.status_code, res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) + return (dict(error=_('Failed to gather reports and maintenance plans from Insights API at URL {}. Server responded with {} status code and message {}').format(url, res.status_code, res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) try: return (dict(insights_content=res.json()), status.HTTP_200_OK) except ValueError: - return (dict(error='Expected JSON response from Insights but instead got {}'.format(res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) + return (dict(error=_('Expected JSON response from Insights but instead got {}').format(res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR) def get(self, request, *args, **kwargs): host = self.get_object() cred = None if host.insights_system_id is None: - return Response(dict(error='This host is not recognized as an Insights host.'), status=status.HTTP_404_NOT_FOUND) + return Response(dict(error=_('This host is not recognized as an Insights host.')), status=status.HTTP_404_NOT_FOUND) - creds = Credential.objects.filter(credential_type__name='Red Hat Satellite 6', credential_type__kind='cloud', credential_type__managed_by_tower=True) - if creds.count() > 0: - cred = creds[0] + if host.inventory and host.inventory.insights_credential: + cred = host.inventory.insights_credential else: - return Response(dict(error='No Insights Credential found for the Inventory, "{}", that this host belongs to.'.format(host.inventory.name)), status=status.HTTP_404_NOT_FOUND) + return Response(dict(error=_('No Insights Credential found for the Inventory, "{}", that this host belongs to.').format(host.inventory.name)), status=status.HTTP_404_NOT_FOUND) - url = 'https://access.redhat.com/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id) + url = settings.INSIGHTS_URL_BASE + '/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id) (username, password) = self._extract_insights_creds(cred) - (msg, status) = self.get_insights(url, username, password) - return Response(msg, status=status) + (msg, err_code) = self.get_insights(url, username, password) + return Response(msg, status=err_code) class GroupList(ListCreateAPIView): diff --git a/awx/main/migrations/0038_v320_release.py b/awx/main/migrations/0038_v320_release.py index afc1e7a1ad..8216282e91 100644 --- a/awx/main/migrations/0038_v320_release.py +++ b/awx/main/migrations/0038_v320_release.py @@ -243,4 +243,14 @@ class Migration(migrations.Migration): name='insights_system_id', field=models.TextField(default=None, help_text='Red Hat Insights host unique identifier.', null=True, db_index=True, blank=True), ), + migrations.AddField( + model_name='inventory', + name='insights_credential', + field=models.ForeignKey(related_name='insights_credential', default=None, blank=True, on_delete=models.deletion.SET_NULL, to='main.Credential', help_text='Credentials to be used by hosts belonging to this invtory when accessing Red Hat Insights API.', null=True), + ), + migrations.AlterField( + model_name='inventory', + name='kind', + field=models.CharField(default=b'', help_text='Kind of inventory being represented.', max_length=32, blank=True, choices=[(b'', 'Hosts have a direct link to this inventory.'), (b'smart', 'Hosts for inventory generated using the host_filter property.')]), + ), ] diff --git a/awx/main/migrations/0040_v320_add_credentialtype_model.py b/awx/main/migrations/0040_v320_add_credentialtype_model.py index 326f9c6d23..9dfef86862 100644 --- a/awx/main/migrations/0040_v320_add_credentialtype_model.py +++ b/awx/main/migrations/0040_v320_add_credentialtype_model.py @@ -23,7 +23,7 @@ class Migration(migrations.Migration): ('modified', models.DateTimeField(default=None, editable=False)), ('description', models.TextField(default=b'', blank=True)), ('name', models.CharField(max_length=512)), - ('kind', models.CharField(max_length=32, choices=[(b'ssh', 'SSH'), (b'vault', 'Vault'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'cloud', 'Cloud')])), + ('kind', models.CharField(max_length=32, choices=[(b'ssh', 'SSH'), (b'vault', 'Vault'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'cloud', 'Cloud'), (b'insights', 'Insights')])), ('managed_by_tower', models.BooleanField(default=False, editable=False)), ('inputs', awx.main.fields.CredentialTypeInputField(default={}, blank=True)), ('injectors', awx.main.fields.CredentialTypeInjectorField(default={}, blank=True)), diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 804ed49798..458635f858 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -399,7 +399,8 @@ class CredentialType(CommonModelNameNotUnique): ('vault', _('Vault')), ('net', _('Network')), ('scm', _('Source Control')), - ('cloud', _('Cloud')) + ('cloud', _('Cloud')), + ('insights', _('Insights')), ) kind = models.CharField( @@ -919,3 +920,32 @@ def azure_rm(cls): }] } ) + + +@CredentialType.default +def insights(cls): + return cls( + kind='insights', + name='Insights Basic Auth', + managed_by_tower=True, + inputs={ + 'fields': [{ + 'id': 'username', + 'label': 'Basic Auth Username', + 'type': 'string' + }, { + 'id': 'password', + 'label': 'Basic Auth Password', + 'type': 'string', + 'secret': True + }], + 'required': ['username', 'password'], + }, + injectors={ + 'extra_vars': { + "scm_username": "{{username}}", + "scm_password": "{{password}}", + }, + }, + ) + diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 4b4f867a44..f1ddb1b1d3 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -143,6 +143,16 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): 'use_role', 'admin_role', ]) + insights_credential = models.ForeignKey( + 'Credential', + related_name='insights_credential', + help_text=_('Credentials to be used by hosts belonging to this invtory when accessing Red Hat Insights API.'), + on_delete=models.SET_NULL, + blank=True, + null=True, + default=None, + ) + def get_absolute_url(self, request=None): return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request) @@ -345,6 +355,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): group_pks = self.groups.values_list('pk', flat=True) return self.groups.exclude(parents__pk__in=group_pks).distinct() + def clean_insights_credential(self): + if self.insights_credential and self.insights_credential.credential_type.kind != 'insights': + raise ValidationError(_('Invalid CredentialType')) + return self.insights_credential + class Host(CommonModelNameNotUnique): ''' diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index 549b6c01bc..7a224c8d43 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -141,7 +141,10 @@ class ProjectOptions(models.Model): return None cred = self.credential if cred: - if cred.kind != 'scm': + if self.scm_type == 'insights': + if cred.kind != 'insights': + raise ValidationError(_("Credential kind must be 'insights'.")) + elif cred.kind != 'scm': raise ValidationError(_("Credential kind must be 'scm'.")) try: if self.scm_type == 'insights': diff --git a/awx/main/tests/functional/api/test_inventory.py b/awx/main/tests/functional/api/test_inventory.py index ddfaa4f562..8f71544f98 100644 --- a/awx/main/tests/functional/api/test_inventory.py +++ b/awx/main/tests/functional/api/test_inventory.py @@ -287,3 +287,17 @@ class TestControlledBySCM: r = options(reverse('api:inventory_inventory_sources_list', kwargs={'pk': scm_inventory.id}), admin_user, expect=200) assert 'POST' not in r.data['actions'] + + +@pytest.mark.django_db +class TestInsightsCredential: + def test_insights_credential(self, patch, insights_inventory, admin_user, insights_credential): + patch(insights_inventory.get_absolute_url(), + {'insights_credential': insights_credential.id}, admin_user, + expect=200) + + def test_non_insights_credential(self, patch, insights_inventory, admin_user, scm_credential): + patch(insights_inventory.get_absolute_url(), + {'insights_credential': scm_credential.id}, admin_user, + expect=400) + diff --git a/awx/main/tests/functional/api/test_project.py b/awx/main/tests/functional/api/test_project.py new file mode 100644 index 0000000000..ddf28f9b52 --- /dev/null +++ b/awx/main/tests/functional/api/test_project.py @@ -0,0 +1,15 @@ +import pytest + + +@pytest.mark.django_db +class TestInsightsCredential: + def test_insights_credential(self, patch, insights_project, admin_user, insights_credential): + patch(insights_project.get_absolute_url(), + {'credential': insights_credential.id}, admin_user, + expect=200) + + def test_non_insights_credential(self, patch, insights_project, admin_user, scm_credential): + patch(insights_project.get_absolute_url(), + {'credential': scm_credential.id}, admin_user, + expect=400) + diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 963466009d..caf03b56e8 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -178,6 +178,11 @@ def user_project(user): return Project.objects.create(name="test-user-project", created_by=owner, description="test-user-project-desc") +@pytest.fixture +def insights_project(): + return Project.objects.create(name="test-insights-project", scm_type="insights") + + @pytest.fixture def instance(settings): return Instance.objects.create(uuid=settings.SYSTEM_UUID, hostname="instance.example.org", capacity=100) @@ -216,6 +221,20 @@ def credentialtype_vault(): return vault_type +@pytest.fixture +def credentialtype_scm(): + scm_type = CredentialType.defaults['scm']() + scm_type.save() + return scm_type + + +@pytest.fixture +def credentialtype_insights(): + insights_type = CredentialType.defaults['insights']() + insights_type.save() + return insights_type + + @pytest.fixture def credential(credentialtype_aws): return Credential.objects.create(credential_type=credentialtype_aws, name='test-cred', @@ -240,6 +259,18 @@ def machine_credential(credentialtype_ssh): inputs={'username': 'test_user', 'password': 'pas4word'}) +@pytest.fixture +def scm_credential(credentialtype_scm): + return Credential.objects.create(credential_type=credentialtype_scm, name='scm-cred', + inputs={'username': 'optimus', 'password': 'prime'}) + + +@pytest.fixture +def insights_credential(credentialtype_insights): + return Credential.objects.create(credential_type=credentialtype_insights, name='insights-cred', + inputs={'username': 'morocco_mole', 'password': 'secret_squirrel'}) + + @pytest.fixture def org_credential(organization, credentialtype_aws): return Credential.objects.create(credential_type=credentialtype_aws, name='test-cred', @@ -252,6 +283,13 @@ def inventory(organization): return organization.inventories.create(name="test-inv") +@pytest.fixture +def insights_inventory(inventory): + inventory.scm_type = 'insights' + inventory.save() + return inventory + + @pytest.fixture def scm_inventory_source(inventory, project): inv_src = InventorySource( diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 0a625b6206..534bd2785f 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -19,6 +19,7 @@ def test_default_cred_types(): 'azure_rm', 'cloudforms', 'gce', + 'insights', 'net', 'openstack', 'satellite6', diff --git a/awx/main/tests/unit/api/test_views.py b/awx/main/tests/unit/api/test_views.py index 837d202044..eb960233cd 100644 --- a/awx/main/tests/unit/api/test_views.py +++ b/awx/main/tests/unit/api/test_views.py @@ -12,6 +12,11 @@ from awx.api.views import ( HostInsights, ) +from awx.main.models import ( + Host, + Inventory, +) + @pytest.fixture def mock_response_new(mocker): @@ -143,10 +148,10 @@ class TestHostInsights(): def test_get_insights_non_200(self, patch_parent, mocker): view = HostInsights() Response = namedtuple('Response', 'status_code content') - mocker.patch.object(view, '_get_insights', return_value=Response(500, 'hello world!')) + mocker.patch.object(view, '_get_insights', return_value=Response(500, 'mock 500 err msg')) (msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore') - assert msg['error'] == 'Failed to gather reports and maintenance plans from Insights API. Server responded with 500 status code and message hello world!' + assert msg['error'] == 'Failed to gather reports and maintenance plans from Insights API at URL https://myexample.com/whocares/me/. Server responded with 500 status code and message mock 500 err msg' def test_get_insights_malformed_json_content(self, patch_parent, mocker): view = HostInsights() @@ -164,3 +169,36 @@ class TestHostInsights(): assert msg['error'] == 'Expected JSON response from Insights but instead got booo!' assert code == 500 + #def test_get_not_insights_host(self, patch_parent, mocker, mock_response_new): + #def test_get_not_insights_host(self, patch_parent, mocker): + def test_get_not_insights_host(self, mocker): + + view = HostInsights() + + host = Host() + host.insights_system_id = None + + mocker.patch.object(view, 'get_object', return_value=host) + + resp = view.get(None) + + assert resp.data['error'] == 'This host is not recognized as an Insights host.' + assert resp.status_code == 404 + + def test_get_no_credential(self, patch_parent, mocker): + view = HostInsights() + + class MockInventory(): + insights_credential = None + name = 'inventory_name_here' + + class MockHost(): + insights_system_id = 'insights_system_id_value' + inventory = MockInventory() + + mocker.patch.object(view, 'get_object', return_value=MockHost()) + + resp = view.get(None) + + assert resp.data['error'] == 'No Insights Credential found for the Inventory, "inventory_name_here", that this host belongs to.' + assert resp.status_code == 404 From 7c0513d5eeb3ddabbe08801d3f548283f2823b16 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 26 May 2017 08:45:35 -0400 Subject: [PATCH 07/10] migrate insights credentials * A credential is considered an insights credential if a project of scm_type 'insights' is associated. * all non-insights Projects that point at an insights credential are nulled --- awx/main/migrations/_credentialtypes.py | 16 +++++++ awx/main/models/credential.py | 2 + .../functional/test_credential_migration.py | 44 +++++++++++++++++-- 3 files changed, 59 insertions(+), 3 deletions(-) diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index a2cc23578f..a51a5fc75b 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -1,6 +1,7 @@ from awx.main import utils from awx.main.models import CredentialType from awx.main.utils.common import encrypt_field, decrypt_field +from django.db.models import Q DEPRECATED_CRED_KIND = { @@ -48,6 +49,14 @@ def _populate_deprecated_cred_types(cred, kind): return cred[kind] +def _is_insights_scm(apps, cred): + return apps.get_model('main', 'Credential').objects.filter(id=cred.id, projects__scm_type='insights').exists() + + +def _disassociate_non_insights_projects(apps, cred): + apps.get_model('main', 'Project').objects.filter(~Q(scm_type='insights') & Q(credential=cred)).update(credential=None) + + def migrate_to_v2_credentials(apps, schema_editor): CredentialType.setup_tower_managed_defaults() deprecated_cred = _generate_deprecated_cred_types() @@ -64,7 +73,11 @@ def migrate_to_v2_credentials(apps, schema_editor): data = {} if getattr(cred, 'vault_password', None): data['vault_password'] = cred.vault_password + if _is_insights_scm(apps, cred): + data['is_insights'] = True + _disassociate_non_insights_projects(apps, cred) credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data) + defined_fields = credential_type.defined_fields cred.credential_type = apps.get_model('main', 'CredentialType').objects.get(pk=credential_type.pk) @@ -80,6 +93,8 @@ def migrate_to_v2_credentials(apps, schema_editor): job.credential = None job.vault_credential = cred job.save() + if data.get('is_insights', False): + cred.kind = 'insights' cred.save() # @@ -145,3 +160,4 @@ def migrate_job_credentials(apps, schema_editor): obj.save() finally: utils.get_current_apps = orig_current_apps + diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 458635f858..5ff69d6fa3 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -469,6 +469,8 @@ class CredentialType(CommonModelNameNotUnique): requirements['kind'] = 'vault' else: requirements['kind'] = 'ssh' + elif kind == 'scm' and data.get('is_insights', False): + requirements['kind'] = 'insights' elif kind in ('net', 'scm'): requirements['kind'] = kind elif kind in kind_choices: diff --git a/awx/main/tests/functional/test_credential_migration.py b/awx/main/tests/functional/test_credential_migration.py index daca6af3df..6595e890e8 100644 --- a/awx/main/tests/functional/test_credential_migration.py +++ b/awx/main/tests/functional/test_credential_migration.py @@ -8,6 +8,7 @@ from django.apps import apps from awx.main.models import Credential, CredentialType from awx.main.migrations._credentialtypes import migrate_to_v2_credentials from awx.main.utils.common import decrypt_field +from awx.main.migrations._credentialtypes import _disassociate_non_insights_projects EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----' @@ -15,12 +16,12 @@ EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY- @contextmanager -def migrate(credential, kind): +def migrate(credential, kind, is_insights=False): with mock.patch.object(Credential, 'kind', kind), \ mock.patch.object(Credential, 'objects', mock.Mock( get=lambda **kw: deepcopy(credential), - all=lambda: [credential] - )): + all=lambda: [credential], + )), mock.patch('awx.main.migrations._credentialtypes._is_insights_scm', return_value=is_insights): class Apps(apps.__class__): def get_model(self, app, model): if model == 'Credential': @@ -307,3 +308,40 @@ def test_azure_rm_migration(): assert decrypt_field(cred, 'secret') == 'some-secret' assert cred.inputs['tenant'] == 'some-tenant' assert Credential.objects.count() == 1 + + +@pytest.mark.django_db +def test_insights_migration(): + cred = Credential(name='My Credential') + + with migrate(cred, 'scm', is_insights=True): + cred.__dict__.update({ + 'username': 'bob', + 'password': 'some-password', + }) + + assert cred.credential_type.name == 'Insights Basic Auth' + assert cred.inputs['username'] == 'bob' + assert cred.inputs['password'].startswith('$encrypted$') + + +@pytest.mark.skip(reason="Need some more mocking here or something.") +@pytest.mark.django_db +def test_insights_project_migration(): + cred1 = apps.get_model('main', 'Credential').objects.create(name='My Credential') + cred2 = apps.get_model('main', 'Credential').objects.create(name='My Credential') + projA1 = apps.get_model('main', 'Project').objects.create(name='Insights Project A1', scm_type='insights', credential=cred1) + + projB1 = apps.get_model('main', 'Project').objects.create(name='Git Project B1', scm_type='git', credential=cred1) + projB2 = apps.get_model('main', 'Project').objects.create(name='Git Project B2', scm_type='git', credential=cred1) + + projC1 = apps.get_model('main', 'Project').objects.create(name='Git Project C1', scm_type='git', credential=cred2) + + _disassociate_non_insights_projects(apps, cred1) + _disassociate_non_insights_projects(apps, cred2) + + assert apps.get_model('main', 'Project').objects.get(pk=projA1).credential is None + assert apps.get_model('main', 'Project').objects.get(pk=projB1).credential is None + assert apps.get_model('main', 'Project').objects.get(pk=projB2).credential is None + assert apps.get_model('main', 'Project').objects.get(pk=projC1).credential == cred2 + From 2f43f0fe2be8b2754a6c5d8a7816e477086f237c Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 26 May 2017 08:52:13 -0400 Subject: [PATCH 08/10] flake8 --- awx/main/tests/unit/api/test_views.py | 1 - 1 file changed, 1 deletion(-) diff --git a/awx/main/tests/unit/api/test_views.py b/awx/main/tests/unit/api/test_views.py index eb960233cd..08f6b9b827 100644 --- a/awx/main/tests/unit/api/test_views.py +++ b/awx/main/tests/unit/api/test_views.py @@ -14,7 +14,6 @@ from awx.api.views import ( from awx.main.models import ( Host, - Inventory, ) From b25b397e1434a4012113006b8d9b856791910f5d Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 26 May 2017 09:58:20 -0400 Subject: [PATCH 09/10] remove insights logic from backwards compat logic * spelling fixes --- awx/main/migrations/0038_v320_release.py | 2 +- awx/main/migrations/_credentialtypes.py | 9 +++++++-- awx/main/models/credential.py | 2 -- awx/main/models/inventory.py | 4 ++-- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/awx/main/migrations/0038_v320_release.py b/awx/main/migrations/0038_v320_release.py index 8216282e91..6b00103df5 100644 --- a/awx/main/migrations/0038_v320_release.py +++ b/awx/main/migrations/0038_v320_release.py @@ -246,7 +246,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='inventory', name='insights_credential', - field=models.ForeignKey(related_name='insights_credential', default=None, blank=True, on_delete=models.deletion.SET_NULL, to='main.Credential', help_text='Credentials to be used by hosts belonging to this invtory when accessing Red Hat Insights API.', null=True), + field=models.ForeignKey(related_name='insights_credential', default=None, blank=True, on_delete=models.deletion.SET_NULL, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True), ), migrations.AlterField( model_name='inventory', diff --git a/awx/main/migrations/_credentialtypes.py b/awx/main/migrations/_credentialtypes.py index a51a5fc75b..4dd083fa0b 100644 --- a/awx/main/migrations/_credentialtypes.py +++ b/awx/main/migrations/_credentialtypes.py @@ -49,6 +49,10 @@ def _populate_deprecated_cred_types(cred, kind): return cred[kind] +def _get_insights_credential_type(): + return CredentialType.objects.get(kind='insights') + + def _is_insights_scm(apps, cred): return apps.get_model('main', 'Credential').objects.filter(id=cred.id, projects__scm_type='insights').exists() @@ -74,9 +78,10 @@ def migrate_to_v2_credentials(apps, schema_editor): if getattr(cred, 'vault_password', None): data['vault_password'] = cred.vault_password if _is_insights_scm(apps, cred): - data['is_insights'] = True _disassociate_non_insights_projects(apps, cred) - credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data) + credential_type = _get_insights_credential_type() + else: + credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data) defined_fields = credential_type.defined_fields cred.credential_type = apps.get_model('main', 'CredentialType').objects.get(pk=credential_type.pk) diff --git a/awx/main/models/credential.py b/awx/main/models/credential.py index 5ff69d6fa3..458635f858 100644 --- a/awx/main/models/credential.py +++ b/awx/main/models/credential.py @@ -469,8 +469,6 @@ class CredentialType(CommonModelNameNotUnique): requirements['kind'] = 'vault' else: requirements['kind'] = 'ssh' - elif kind == 'scm' and data.get('is_insights', False): - requirements['kind'] = 'insights' elif kind in ('net', 'scm'): requirements['kind'] = kind elif kind in kind_choices: diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index f1ddb1b1d3..8785aba6fc 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -146,7 +146,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): insights_credential = models.ForeignKey( 'Credential', related_name='insights_credential', - help_text=_('Credentials to be used by hosts belonging to this invtory when accessing Red Hat Insights API.'), + help_text=_('Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.'), on_delete=models.SET_NULL, blank=True, null=True, @@ -357,7 +357,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): def clean_insights_credential(self): if self.insights_credential and self.insights_credential.credential_type.kind != 'insights': - raise ValidationError(_('Invalid CredentialType')) + raise ValidationError(_("Credential kind must be 'insights'.")) return self.insights_credential From 441946d33233447e9808e63b1f35e3214dd6cc35 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Fri, 26 May 2017 14:04:15 -0400 Subject: [PATCH 10/10] add v1 vs. v2 inclusion/exclusion --- awx/api/serializers.py | 3 ++- awx/api/views.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 57df11ef09..1c3ac0d3fc 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1208,8 +1208,9 @@ class HostSerializer(BaseSerializerWithVariables): ad_hoc_commands = self.reverse('api:host_ad_hoc_commands_list', kwargs={'pk': obj.pk}), ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}), fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}), - insights = self.reverse('api:host_insights', kwargs={'pk': obj.pk}), )) + if self.version > 1: + res['insights'] = self.reverse('api:host_insights', kwargs={'pk': obj.pk}) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) if obj.last_job: diff --git a/awx/api/views.py b/awx/api/views.py index 89b682bdbd..ff08d120cb 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -2073,6 +2073,8 @@ class HostInsights(GenericAPIView): model = Host serializer_class = EmptySerializer + new_in_320 = True + new_in_api_v2 = True def _extract_insights_creds(self, credential): return (credential.inputs['username'], decrypt_field(credential, 'password'))