Merge pull request #6300 from chrismeyersfsu/feature-insights_proxy

insights proxy
This commit is contained in:
Chris Meyers 2017-05-26 14:17:04 -04:00 committed by GitHub
commit 1a10205e0c
16 changed files with 346 additions and 18 deletions

View File

@ -1114,7 +1114,8 @@ class InventorySerializer(BaseSerializerWithVariables):
fields = ('*', 'organization', 'kind', 'host_filter', 'variables', 'has_active_failures',
'total_hosts', 'hosts_with_active_failures', 'total_groups',
'groups_with_active_failures', 'has_inventory_sources',
'total_inventory_sources', 'inventory_sources_with_failures')
'total_inventory_sources', 'inventory_sources_with_failures',
'insights_credential',)
def get_related(self, obj):
res = super(InventorySerializer, self).get_related(obj)
@ -1135,6 +1136,8 @@ class InventorySerializer(BaseSerializerWithVariables):
object_roles = self.reverse('api:inventory_object_roles_list', kwargs={'pk': obj.pk}),
instance_groups = self.reverse('api:inventory_instance_groups_list', kwargs={'pk': obj.pk}),
))
if obj.insights_credential:
res['insights_credential'] = self.reverse('api:credential_detail', kwargs={'pk': obj.insights_credential.pk})
if obj.organization:
res['organization'] = self.reverse('api:organization_detail', kwargs={'pk': obj.organization.pk})
return res
@ -1208,6 +1211,8 @@ class HostSerializer(BaseSerializerWithVariables):
ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}),
fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}),
))
if self.version > 1:
res['insights'] = self.reverse('api:host_insights', kwargs={'pk': obj.pk})
if obj.inventory:
res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
if obj.last_job:

View File

@ -120,6 +120,7 @@ host_urls = patterns('awx.api.views',
#url(r'^(?P<pk>[0-9]+)/single_fact/$', 'host_single_fact_view'),
url(r'^(?P<pk>[0-9]+)/fact_versions/$', 'host_fact_versions_list'),
url(r'^(?P<pk>[0-9]+)/fact_view/$', 'host_fact_compare_view'),
url(r'^(?P<pk>[0-9]+)/insights/$', 'host_insights'),
)
group_urls = patterns('awx.api.views',

View File

@ -13,6 +13,7 @@ import socket
import subprocess
import sys
import logging
import requests
from base64 import b64encode
from collections import OrderedDict
@ -70,7 +71,8 @@ from awx.conf.license import get_license, feature_enabled, feature_exists, Licen
from awx.main.models import * # noqa
from awx.main.utils import * # noqa
from awx.main.utils import (
callback_filter_out_ansible_extra_vars
callback_filter_out_ansible_extra_vars,
decrypt_field,
)
from awx.main.utils.filters import SmartFilter
@ -2067,6 +2069,58 @@ class HostFactCompareView(SystemTrackingEnforcementMixin, SubDetailAPIView):
return Response(self.serializer_class(instance=fact_entry).data)
class HostInsights(GenericAPIView):
model = Host
serializer_class = EmptySerializer
new_in_320 = True
new_in_api_v2 = True
def _extract_insights_creds(self, credential):
return (credential.inputs['username'], decrypt_field(credential, 'password'))
def _get_insights(self, url, username, password):
session = requests.Session()
session.auth = requests.auth.HTTPBasicAuth(username, password)
headers = {'Content-Type': 'application/json'}
return session.get(url, headers=headers, timeout=120)
def get_insights(self, url, username, password):
try:
res = self._get_insights(url, username, password)
except requests.exceptions.SSLError:
return (dict(error=_('SSLError while trying to connect to {}').format(url)), status.HTTP_500_INTERNAL_SERVER_ERROR)
except requests.exceptions.Timeout:
return (dict(error=_('Request to {} timed out.').format(url)), status.HTTP_504_GATEWAY_TIMEOUT)
except requests.exceptions.RequestException as e:
return (dict(error=_('Unkown exception {} while trying to GET {}').format(e, url)), status.HTTP_500_INTERNAL_SERVER_ERROR)
if res.status_code != 200:
return (dict(error=_('Failed to gather reports and maintenance plans from Insights API at URL {}. Server responded with {} status code and message {}').format(url, res.status_code, res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR)
try:
return (dict(insights_content=res.json()), status.HTTP_200_OK)
except ValueError:
return (dict(error=_('Expected JSON response from Insights but instead got {}').format(res.content)), status.HTTP_500_INTERNAL_SERVER_ERROR)
def get(self, request, *args, **kwargs):
host = self.get_object()
cred = None
if host.insights_system_id is None:
return Response(dict(error=_('This host is not recognized as an Insights host.')), status=status.HTTP_404_NOT_FOUND)
if host.inventory and host.inventory.insights_credential:
cred = host.inventory.insights_credential
else:
return Response(dict(error=_('No Insights Credential found for the Inventory, "{}", that this host belongs to.').format(host.inventory.name)), status=status.HTTP_404_NOT_FOUND)
url = settings.INSIGHTS_URL_BASE + '/r/insights/v3/systems/{}/reports/'.format(host.insights_system_id)
(username, password) = self._extract_insights_creds(cred)
(msg, err_code) = self.get_insights(url, username, password)
return Response(msg, status=err_code)
class GroupList(ListCreateAPIView):
model = Group

View File

@ -243,4 +243,14 @@ class Migration(migrations.Migration):
name='insights_system_id',
field=models.TextField(default=None, help_text='Red Hat Insights host unique identifier.', null=True, db_index=True, blank=True),
),
migrations.AddField(
model_name='inventory',
name='insights_credential',
field=models.ForeignKey(related_name='insights_credential', default=None, blank=True, on_delete=models.deletion.SET_NULL, to='main.Credential', help_text='Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.', null=True),
),
migrations.AlterField(
model_name='inventory',
name='kind',
field=models.CharField(default=b'', help_text='Kind of inventory being represented.', max_length=32, blank=True, choices=[(b'', 'Hosts have a direct link to this inventory.'), (b'smart', 'Hosts for inventory generated using the host_filter property.')]),
),
]

View File

@ -23,7 +23,7 @@ class Migration(migrations.Migration):
('modified', models.DateTimeField(default=None, editable=False)),
('description', models.TextField(default=b'', blank=True)),
('name', models.CharField(max_length=512)),
('kind', models.CharField(max_length=32, choices=[(b'ssh', 'SSH'), (b'vault', 'Vault'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'cloud', 'Cloud')])),
('kind', models.CharField(max_length=32, choices=[(b'ssh', 'SSH'), (b'vault', 'Vault'), (b'net', 'Network'), (b'scm', 'Source Control'), (b'cloud', 'Cloud'), (b'insights', 'Insights')])),
('managed_by_tower', models.BooleanField(default=False, editable=False)),
('inputs', awx.main.fields.CredentialTypeInputField(default={}, blank=True)),
('injectors', awx.main.fields.CredentialTypeInjectorField(default={}, blank=True)),

View File

@ -1,6 +1,7 @@
from awx.main import utils
from awx.main.models import CredentialType
from awx.main.utils.common import encrypt_field, decrypt_field
from django.db.models import Q
DEPRECATED_CRED_KIND = {
@ -48,6 +49,18 @@ def _populate_deprecated_cred_types(cred, kind):
return cred[kind]
def _get_insights_credential_type():
return CredentialType.objects.get(kind='insights')
def _is_insights_scm(apps, cred):
return apps.get_model('main', 'Credential').objects.filter(id=cred.id, projects__scm_type='insights').exists()
def _disassociate_non_insights_projects(apps, cred):
apps.get_model('main', 'Project').objects.filter(~Q(scm_type='insights') & Q(credential=cred)).update(credential=None)
def migrate_to_v2_credentials(apps, schema_editor):
CredentialType.setup_tower_managed_defaults()
deprecated_cred = _generate_deprecated_cred_types()
@ -64,7 +77,12 @@ def migrate_to_v2_credentials(apps, schema_editor):
data = {}
if getattr(cred, 'vault_password', None):
data['vault_password'] = cred.vault_password
credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data)
if _is_insights_scm(apps, cred):
_disassociate_non_insights_projects(apps, cred)
credential_type = _get_insights_credential_type()
else:
credential_type = _populate_deprecated_cred_types(deprecated_cred, cred.kind) or CredentialType.from_v1_kind(cred.kind, data)
defined_fields = credential_type.defined_fields
cred.credential_type = apps.get_model('main', 'CredentialType').objects.get(pk=credential_type.pk)
@ -80,6 +98,8 @@ def migrate_to_v2_credentials(apps, schema_editor):
job.credential = None
job.vault_credential = cred
job.save()
if data.get('is_insights', False):
cred.kind = 'insights'
cred.save()
#
@ -145,3 +165,4 @@ def migrate_job_credentials(apps, schema_editor):
obj.save()
finally:
utils.get_current_apps = orig_current_apps

View File

@ -407,7 +407,8 @@ class CredentialType(CommonModelNameNotUnique):
('vault', _('Vault')),
('net', _('Network')),
('scm', _('Source Control')),
('cloud', _('Cloud'))
('cloud', _('Cloud')),
('insights', _('Insights')),
)
kind = models.CharField(
@ -927,3 +928,32 @@ def azure_rm(cls):
}]
}
)
@CredentialType.default
def insights(cls):
return cls(
kind='insights',
name='Insights Basic Auth',
managed_by_tower=True,
inputs={
'fields': [{
'id': 'username',
'label': 'Basic Auth Username',
'type': 'string'
}, {
'id': 'password',
'label': 'Basic Auth Password',
'type': 'string',
'secret': True
}],
'required': ['username', 'password'],
},
injectors={
'extra_vars': {
"scm_username": "{{username}}",
"scm_password": "{{password}}",
},
},
)

View File

@ -143,6 +143,16 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
'use_role',
'admin_role',
])
insights_credential = models.ForeignKey(
'Credential',
related_name='insights_credential',
help_text=_('Credentials to be used by hosts belonging to this inventory when accessing Red Hat Insights API.'),
on_delete=models.SET_NULL,
blank=True,
null=True,
default=None,
)
def get_absolute_url(self, request=None):
return reverse('api:inventory_detail', kwargs={'pk': self.pk}, request=request)
@ -345,6 +355,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
group_pks = self.groups.values_list('pk', flat=True)
return self.groups.exclude(parents__pk__in=group_pks).distinct()
def clean_insights_credential(self):
if self.insights_credential and self.insights_credential.credential_type.kind != 'insights':
raise ValidationError(_("Credential kind must be 'insights'."))
return self.insights_credential
class Host(CommonModelNameNotUnique):
'''

View File

@ -141,7 +141,10 @@ class ProjectOptions(models.Model):
return None
cred = self.credential
if cred:
if cred.kind != 'scm':
if self.scm_type == 'insights':
if cred.kind != 'insights':
raise ValidationError(_("Credential kind must be 'insights'."))
elif cred.kind != 'scm':
raise ValidationError(_("Credential kind must be 'scm'."))
try:
if self.scm_type == 'insights':

View File

@ -287,3 +287,17 @@ class TestControlledBySCM:
r = options(reverse('api:inventory_inventory_sources_list', kwargs={'pk': scm_inventory.id}),
admin_user, expect=200)
assert 'POST' not in r.data['actions']
@pytest.mark.django_db
class TestInsightsCredential:
def test_insights_credential(self, patch, insights_inventory, admin_user, insights_credential):
patch(insights_inventory.get_absolute_url(),
{'insights_credential': insights_credential.id}, admin_user,
expect=200)
def test_non_insights_credential(self, patch, insights_inventory, admin_user, scm_credential):
patch(insights_inventory.get_absolute_url(),
{'insights_credential': scm_credential.id}, admin_user,
expect=400)

View File

@ -0,0 +1,15 @@
import pytest
@pytest.mark.django_db
class TestInsightsCredential:
def test_insights_credential(self, patch, insights_project, admin_user, insights_credential):
patch(insights_project.get_absolute_url(),
{'credential': insights_credential.id}, admin_user,
expect=200)
def test_non_insights_credential(self, patch, insights_project, admin_user, scm_credential):
patch(insights_project.get_absolute_url(),
{'credential': scm_credential.id}, admin_user,
expect=400)

View File

@ -178,6 +178,11 @@ def user_project(user):
return Project.objects.create(name="test-user-project", created_by=owner, description="test-user-project-desc")
@pytest.fixture
def insights_project():
return Project.objects.create(name="test-insights-project", scm_type="insights")
@pytest.fixture
def instance(settings):
return Instance.objects.create(uuid=settings.SYSTEM_UUID, hostname="instance.example.org", capacity=100)
@ -216,6 +221,20 @@ def credentialtype_vault():
return vault_type
@pytest.fixture
def credentialtype_scm():
scm_type = CredentialType.defaults['scm']()
scm_type.save()
return scm_type
@pytest.fixture
def credentialtype_insights():
insights_type = CredentialType.defaults['insights']()
insights_type.save()
return insights_type
@pytest.fixture
def credential(credentialtype_aws):
return Credential.objects.create(credential_type=credentialtype_aws, name='test-cred',
@ -240,6 +259,18 @@ def machine_credential(credentialtype_ssh):
inputs={'username': 'test_user', 'password': 'pas4word'})
@pytest.fixture
def scm_credential(credentialtype_scm):
return Credential.objects.create(credential_type=credentialtype_scm, name='scm-cred',
inputs={'username': 'optimus', 'password': 'prime'})
@pytest.fixture
def insights_credential(credentialtype_insights):
return Credential.objects.create(credential_type=credentialtype_insights, name='insights-cred',
inputs={'username': 'morocco_mole', 'password': 'secret_squirrel'})
@pytest.fixture
def org_credential(organization, credentialtype_aws):
return Credential.objects.create(credential_type=credentialtype_aws, name='test-cred',
@ -252,6 +283,13 @@ def inventory(organization):
return organization.inventories.create(name="test-inv")
@pytest.fixture
def insights_inventory(inventory):
inventory.scm_type = 'insights'
inventory.save()
return inventory
@pytest.fixture
def scm_inventory_source(inventory, project):
inv_src = InventorySource(

View File

@ -19,6 +19,7 @@ def test_default_cred_types():
'azure_rm',
'cloudforms',
'gce',
'insights',
'net',
'openstack',
'satellite6',

View File

@ -8,6 +8,7 @@ from django.apps import apps
from awx.main.models import Credential, CredentialType
from awx.main.migrations._credentialtypes import migrate_to_v2_credentials
from awx.main.utils.common import decrypt_field
from awx.main.migrations._credentialtypes import _disassociate_non_insights_projects
EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-----'
@ -15,12 +16,12 @@ EXAMPLE_PRIVATE_KEY = '-----BEGIN PRIVATE KEY-----\nxyz==\n-----END PRIVATE KEY-
@contextmanager
def migrate(credential, kind):
def migrate(credential, kind, is_insights=False):
with mock.patch.object(Credential, 'kind', kind), \
mock.patch.object(Credential, 'objects', mock.Mock(
get=lambda **kw: deepcopy(credential),
all=lambda: [credential]
)):
all=lambda: [credential],
)), mock.patch('awx.main.migrations._credentialtypes._is_insights_scm', return_value=is_insights):
class Apps(apps.__class__):
def get_model(self, app, model):
if model == 'Credential':
@ -307,3 +308,40 @@ def test_azure_rm_migration():
assert decrypt_field(cred, 'secret') == 'some-secret'
assert cred.inputs['tenant'] == 'some-tenant'
assert Credential.objects.count() == 1
@pytest.mark.django_db
def test_insights_migration():
cred = Credential(name='My Credential')
with migrate(cred, 'scm', is_insights=True):
cred.__dict__.update({
'username': 'bob',
'password': 'some-password',
})
assert cred.credential_type.name == 'Insights Basic Auth'
assert cred.inputs['username'] == 'bob'
assert cred.inputs['password'].startswith('$encrypted$')
@pytest.mark.skip(reason="Need some more mocking here or something.")
@pytest.mark.django_db
def test_insights_project_migration():
cred1 = apps.get_model('main', 'Credential').objects.create(name='My Credential')
cred2 = apps.get_model('main', 'Credential').objects.create(name='My Credential')
projA1 = apps.get_model('main', 'Project').objects.create(name='Insights Project A1', scm_type='insights', credential=cred1)
projB1 = apps.get_model('main', 'Project').objects.create(name='Git Project B1', scm_type='git', credential=cred1)
projB2 = apps.get_model('main', 'Project').objects.create(name='Git Project B2', scm_type='git', credential=cred1)
projC1 = apps.get_model('main', 'Project').objects.create(name='Git Project C1', scm_type='git', credential=cred2)
_disassociate_non_insights_projects(apps, cred1)
_disassociate_non_insights_projects(apps, cred2)
assert apps.get_model('main', 'Project').objects.get(pk=projA1).credential is None
assert apps.get_model('main', 'Project').objects.get(pk=projB1).credential is None
assert apps.get_model('main', 'Project').objects.get(pk=projB2).credential is None
assert apps.get_model('main', 'Project').objects.get(pk=projC1).credential == cred2

View File

@ -1,5 +1,6 @@
import mock
import pytest
import requests
from collections import namedtuple
@ -8,6 +9,11 @@ from awx.api.views import (
JobTemplateLabelList,
JobTemplateSurveySpec,
InventoryInventorySourcesUpdate,
HostInsights,
)
from awx.main.models import (
Host,
)
@ -117,3 +123,81 @@ class TestInventoryInventorySourcesUpdate:
view = InventoryInventorySourcesUpdate()
response = view.post(mock_request)
assert response.data == expected
class TestHostInsights():
@pytest.fixture
def patch_parent(self, mocker):
mocker.patch('awx.api.generics.GenericAPIView')
@pytest.mark.parametrize("status_code, exception, error, message", [
(500, requests.exceptions.SSLError, 'SSLError while trying to connect to https://myexample.com/whocares/me/', None,),
(504, requests.exceptions.Timeout, 'Request to https://myexample.com/whocares/me/ timed out.', None,),
(500, requests.exceptions.RequestException, 'booo!', 'Unkown exception booo! while trying to GET https://myexample.com/whocares/me/'),
])
def test_get_insights_request_exception(self, patch_parent, mocker, status_code, exception, error, message):
view = HostInsights()
mocker.patch.object(view, '_get_insights', side_effect=exception(error))
(msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore')
assert code == status_code
assert msg['error'] == message or error
def test_get_insights_non_200(self, patch_parent, mocker):
view = HostInsights()
Response = namedtuple('Response', 'status_code content')
mocker.patch.object(view, '_get_insights', return_value=Response(500, 'mock 500 err msg'))
(msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore')
assert msg['error'] == 'Failed to gather reports and maintenance plans from Insights API at URL https://myexample.com/whocares/me/. Server responded with 500 status code and message mock 500 err msg'
def test_get_insights_malformed_json_content(self, patch_parent, mocker):
view = HostInsights()
class Response():
status_code = 200
content = 'booo!'
def json(self):
raise ValueError('we do not care what this is')
mocker.patch.object(view, '_get_insights', return_value=Response())
(msg, code) = view.get_insights('https://myexample.com/whocares/me/', 'ignore', 'ignore')
assert msg['error'] == 'Expected JSON response from Insights but instead got booo!'
assert code == 500
#def test_get_not_insights_host(self, patch_parent, mocker, mock_response_new):
#def test_get_not_insights_host(self, patch_parent, mocker):
def test_get_not_insights_host(self, mocker):
view = HostInsights()
host = Host()
host.insights_system_id = None
mocker.patch.object(view, 'get_object', return_value=host)
resp = view.get(None)
assert resp.data['error'] == 'This host is not recognized as an Insights host.'
assert resp.status_code == 404
def test_get_no_credential(self, patch_parent, mocker):
view = HostInsights()
class MockInventory():
insights_credential = None
name = 'inventory_name_here'
class MockHost():
insights_system_id = 'insights_system_id_value'
inventory = MockInventory()
mocker.patch.object(view, 'get_object', return_value=MockHost())
resp = view.get(None)
assert resp.data['error'] == 'No Insights Credential found for the Inventory, "inventory_name_here", that this host belongs to.'
assert resp.status_code == 404

View File

@ -1,14 +1,13 @@
#!/usr/bin/env python
from ansible.module_utils.basic import * # noqa
import uuid
DOCUMENTATION = '''
---
module: scan_insights
short_description: Return insights UUID as fact data
short_description: Return insights id as fact data
description:
- Inspects the /etc/redhat-access-insights/machine-id file for insights uuid and returns the found UUID as fact data
- Inspects the /etc/redhat-access-insights/machine-id file for insights id and returns the found id as fact data
version_added: "2.3"
options:
requirements: [ ]
@ -28,8 +27,8 @@ EXAMPLES = '''
INSIGHTS_SYSTEM_ID_FILE='/etc/redhat-access-insights/machine-id'
def get_system_uuid(filname):
system_uuid = None
def get_system_id(filname):
system_id = None
try:
f = open(INSIGHTS_SYSTEM_ID_FILE, "r")
except IOError:
@ -37,12 +36,12 @@ def get_system_uuid(filname):
else:
try:
data = f.readline()
system_uuid = str(uuid.UUID(data))
system_id = str(data)
except (IOError, ValueError):
pass
finally:
f.close()
return system_uuid
return system_id
def main():
@ -50,12 +49,12 @@ def main():
argument_spec = dict()
)
system_uuid = get_system_uuid(INSIGHTS_SYSTEM_ID_FILE)
system_id = get_system_id(INSIGHTS_SYSTEM_ID_FILE)
results = {
'ansible_facts': {
'insights': {
'system_id': system_uuid
'system_id': system_id
}
}
}