diff --git a/.coveragerc b/.coveragerc index c58219e418..d7a734c1d6 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,7 +1,9 @@ [run] -source = ansibleworks +source = awx branch = True -omit = ansibleworks/main/migrations/* +omit = + awx/main/migrations/* + awx/lib/site-packages/* [report] # Regexes for lines to exclude from consideration diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 528545da30..daab3fac03 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -412,6 +412,14 @@ class Credential(CommonModelNameNotUnique): @property def needs_sudo_password(self): return self.sudo_password == 'ASK' + + @property + def passwords_needed(self): + needed = [] + for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'): + if getattr(self, 'needs_%s' % field): + needed.append(field) + return needed def get_absolute_url(self): return reverse('main:credential_detail', args=(self.pk,)) @@ -646,14 +654,11 @@ class JobTemplate(CommonModel): return reverse('main:job_template_detail', args=(self.pk,)) def can_start_without_user_input(self): - '''Return whether job template can be used to start a new job without - requiring any user input.''' - if not self.credential: - return False - for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'): - if getattr(self.credential, 'needs_%s' % field): - return False - return True + ''' + Return whether job template can be used to start a new job without + requiring any user input. + ''' + return bool(self.credential and not self.credential.passwords_needed) class Job(CommonModel): ''' @@ -832,11 +837,7 @@ class Job(CommonModel): def get_passwords_needed_to_start(self): '''Return list of password field names needed to start the job.''' - needed = [] - for field in ('ssh_password', 'sudo_password', 'ssh_key_unlock'): - if self.credential and getattr(self.credential, 'needs_%s' % field): - needed.append(field) - return needed + return (self.credential and self.credential.passwords_needed) or [] @property def can_start(self): diff --git a/awx/main/rbac.py b/awx/main/rbac.py index bc6ddc8de0..979b592904 100644 --- a/awx/main/rbac.py +++ b/awx/main/rbac.py @@ -127,36 +127,46 @@ class JobTemplateCallbackPermission(CustomRbac): def has_permission(self, request, view, obj=None): # If another authentication method was used and it's not a POST, return # True to fall through to the next permission class. - if request.user or request.auth and request.method.lower() != 'post': + if (request.user or request.auth) and request.method.lower() != 'post': return super(JobTemplateCallbackPermission, self).has_permission(request, view, obj) - return False - # FIXME - #try: - # job_template = JobTemplate.objects.get(active=True, pk=int(request.auth.split('-')[0])) - #except Job.DoesNotExist: - # return False + # Require method to be POST, host_config_key to be specified and match + # the requested job template, and require the job template to be + # active in order to proceed. + host_config_key = request.DATA.get('host_config_key', '') + if request.method.lower() != 'post': + return False + elif not host_config_key: + return False + elif obj and not obj.active: + return False + elif obj and obj.host_config_key != host_config_key: + return False + else: + return True class JobTaskPermission(CustomRbac): def has_permission(self, request, view, obj=None): - # If another authentication method was used other than the one for job - # callbacks, return True to fall through to the next permission class. + # callbacks, default to the superclass permissions checking. if request.user or not request.auth: return super(JobTaskPermission, self).has_permission(request, view, obj) - # FIXME: Verify that inventory or job event requested are for the same - # job ID present in the auth token, etc. - - #try: - # job = Job.objects.get(active=True, status='running', pk=int(request.auth.split('-')[0])) - #except Job.DoesNotExist: - # return False + # Verify that the job ID present in the auth token is for a valid, + # active job. + try: + job = Job.objects.get(active=True, status='running', + pk=int(request.auth.split('-')[0])) + except (Job.DoesNotExist, TypeError): + return False + # Verify that the request method is one of those allowed for the given + # view, also that the job or inventory being accessed matches the auth + # token. if view.model == Inventory and request.method.lower() in ('head', 'get'): - return True + return bool(not obj or obj.pk == job.inventory.pk) elif view.model == JobEvent and request.method.lower() == 'post': - return True + return bool(not obj or obj.pk == job.pk) else: return False diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 6c72fdaf40..dc84f261d8 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -38,6 +38,7 @@ class RunJob(Task): if field == 'status': update_fields.append('failed') job.save(update_fields=update_fields) + # FIXME: Commit transaction? return job def get_path_to(self, *args): diff --git a/awx/main/tests/base.py b/awx/main/tests/base.py index 6784cc5ab6..143985fa0e 100644 --- a/awx/main/tests/base.py +++ b/awx/main/tests/base.py @@ -145,7 +145,7 @@ class BaseTestMixin(object): return ('random', 'combination') def _generic_rest(self, url, data=None, expect=204, auth=None, method=None, - data_type=None, accept=None): + data_type=None, accept=None, remote_addr=None): assert method is not None method_name = method.lower() if method_name not in ('options', 'head', 'get', 'delete'): @@ -153,6 +153,8 @@ class BaseTestMixin(object): client_kwargs = {} if accept: client_kwargs['HTTP_ACCEPT'] = accept + if remote_addr: + client_kwargs['REMOTE_ADDR'] = remote_addr client = Client(**client_kwargs) auth = auth or self._current_auth if auth: @@ -191,39 +193,46 @@ class BaseTestMixin(object): else: return None - def options(self, url, expect=200, auth=None, accept=None): + def options(self, url, expect=200, auth=None, accept=None, + remote_addr=None): return self._generic_rest(url, data=None, expect=expect, auth=auth, - method='options', accept=accept) + method='options', accept=accept, + remote_addr=remote_addr) - def head(self, url, expect=200, auth=None, accept=None): + def head(self, url, expect=200, auth=None, accept=None, remote_addr=None): return self._generic_rest(url, data=None, expect=expect, auth=auth, - method='head', accept=accept) + method='head', accept=accept, + remote_addr=remote_addr) - def get(self, url, expect=200, auth=None, accept=None): + def get(self, url, expect=200, auth=None, accept=None, remote_addr=None): return self._generic_rest(url, data=None, expect=expect, auth=auth, - method='get', accept=accept) + method='get', accept=accept, + remote_addr=remote_addr) def post(self, url, data, expect=204, auth=None, data_type=None, - accept=None): + accept=None, remote_addr=None): return self._generic_rest(url, data=data, expect=expect, auth=auth, method='post', data_type=data_type, - accept=accept) + accept=accept, + remote_addr=remote_addr) def put(self, url, data, expect=200, auth=None, data_type=None, - accept=None): + accept=None, remote_addr=None): return self._generic_rest(url, data=data, expect=expect, auth=auth, method='put', data_type=data_type, - accept=accept) + accept=accept, remote_addr=remote_addr) def patch(self, url, data, expect=200, auth=None, data_type=None, - accept=None): + accept=None, remote_addr=None): return self._generic_rest(url, data=data, expect=expect, auth=auth, method='patch', data_type=data_type, - accept=accept) + accept=accept, remote_addr=remote_addr) - def delete(self, url, expect=201, auth=None, data_type=None, accept=None): + def delete(self, url, expect=201, auth=None, data_type=None, accept=None, + remote_addr=None): return self._generic_rest(url, data=None, expect=expect, auth=auth, - method='delete', accept=accept) + method='delete', accept=accept, + remote_addr=remote_addr) def get_urls(self, collection_url, auth=None): # TODO: this test helper function doesn't support pagination diff --git a/awx/main/tests/jobs.py b/awx/main/tests/jobs.py index 046d0a93fa..8c7c954842 100644 --- a/awx/main/tests/jobs.py +++ b/awx/main/tests/jobs.py @@ -1,18 +1,27 @@ # Copyright (c) 2013 AnsibleWorks, Inc. # All Rights Reserved. +# Python import datetime import json +import socket +import struct +import uuid + +# Django from django.contrib.auth.models import User as DjangoUser from django.core.urlresolvers import reverse from django.db import transaction import django.test from django.test.client import Client from django.test.utils import override_settings + +# AWX from awx.main.models import * from awx.main.tests.base import BaseTestMixin -__all__ = ['JobTemplateTest', 'JobTest', 'JobStartCancelTest'] +__all__ = ['JobTemplateTest', 'JobTest', 'JobStartCancelTest', + 'JobTemplateCallbackTest'] TEST_PLAYBOOK = '''- hosts: all gather_facts: false @@ -308,6 +317,7 @@ class BaseJobTestMixin(BaseTestMixin): inventory= self.inv_eng, project=self.proj_dev, playbook=self.proj_dev.playbooks[0], + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_eng_check = self.jt_eng_check.create_job( @@ -320,6 +330,7 @@ class BaseJobTestMixin(BaseTestMixin): inventory= self.inv_eng, project=self.proj_dev, playbook=self.proj_dev.playbooks[0], + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_eng_run = self.jt_eng_run.create_job( @@ -335,6 +346,7 @@ class BaseJobTestMixin(BaseTestMixin): inventory= self.inv_sup, project=self.proj_test, playbook=self.proj_test.playbooks[0], + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_sup_check = self.jt_sup_check.create_job( @@ -347,6 +359,7 @@ class BaseJobTestMixin(BaseTestMixin): inventory= self.inv_sup, project=self.proj_test, playbook=self.proj_test.playbooks[0], + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_sup_run = self.jt_sup_run.create_job( @@ -363,6 +376,7 @@ class BaseJobTestMixin(BaseTestMixin): project=self.proj_prod, playbook=self.proj_prod.playbooks[0], credential=self.cred_ops_east, + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_ops_east_check = self.jt_ops_east_check.create_job( @@ -375,6 +389,7 @@ class BaseJobTestMixin(BaseTestMixin): project=self.proj_prod, playbook=self.proj_prod.playbooks[0], credential=self.cred_ops_east, + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_ops_east_run = self.jt_ops_east_run.create_job( @@ -387,6 +402,7 @@ class BaseJobTestMixin(BaseTestMixin): project=self.proj_prod, playbook=self.proj_prod.playbooks[0], credential=self.cred_ops_west, + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_ops_west_check = self.jt_ops_west_check.create_job( @@ -399,6 +415,7 @@ class BaseJobTestMixin(BaseTestMixin): project=self.proj_prod, playbook=self.proj_prod.playbooks[0], credential=self.cred_ops_west, + host_config_key=uuid.uuid4().hex, created_by=self.user_sue, ) self.job_ops_west_run = self.jt_ops_west_run.create_job( @@ -965,3 +982,142 @@ class JobStartCancelTest(BaseJobTestMixin, django.test.LiveServerTestCase): self.assertTrue(qs.count()) self.check_pagination_and_size(response, qs.count()) self.check_list_ids(response, qs) + +@override_settings(CELERY_ALWAYS_EAGER=True, + CELERY_EAGER_PROPAGATES_EXCEPTIONS=True, + ANSIBLE_TRANSPORT='local', + MIDDLEWARE_CLASSES=MIDDLEWARE_CLASSES) +class JobTemplateCallbackTest(BaseJobTestMixin, django.test.LiveServerTestCase): + '''Job template callback tests for empheral hosts.''' + + def setUp(self): + super(JobTemplateCallbackTest, self).setUp() + settings.INTERNAL_API_URL = self.live_server_url + # Monkeypatch socket module DNS lookup functions for testing. + self._original_gethostbyaddr = socket.gethostbyaddr + self._original_getaddrinfo = socket.getaddrinfo + socket.gethostbyaddr = self.gethostbyaddr + socket.getaddrinfo = self.getaddrinfo + + def tearDown(self): + super(JobTemplateCallbackTest, self).tearDown() + socket.gethostbyaddr = self._original_gethostbyaddr + socket.getaddrinfo = self._original_getaddrinfo + + def atoh(self, a): + '''Convert IP address to integer in host byte order.''' + return socket.ntohl(struct.unpack('I', socket.inet_aton(a))[0]) + + def htoa(self, n): + '''Convert integer in host byte order to IP address.''' + return socket.inet_ntoa(struct.pack('I', socket.htonl(n))) + + def get_test_ips_for_host(self, host): + '''Return test IP address(es) for given test hostname.''' + ips = [] + try: + h = Host.objects.get(name=host) + # Primary IP for host (both forward/reverse lookups work). + val = self.atoh('127.10.0.0') + h.pk + ips.append(self.htoa(val)) + # Secondary IP for host (both forward/reverse lookups work). + if h.pk % 2 == 0: + val = self.atoh('127.20.0.0') + h.pk + ips.append(self.htoa(val)) + # Additional IP for host (only forward lookups work). + if h.pk % 3 == 0: + val = self.atoh('127.30.0.0') + h.pk + ips.append(self.htoa(val)) + except Host.DoesNotExist: + pass + return ips + + def get_test_host_for_ip(self, ip): + '''Return test hostname for given test IP address.''' + if not ip.startswith('127.10.') and not ip.startswith('127.20.'): + return None + val = self.atoh(ip) + try: + return Host.objects.get(pk=(val & 0x0ffff)).name + except Host.DoesNotExist: + return None + + def test_dummy_host_ip_lookup(self): + all_ips = set() + for host in Host.objects.all(): + ips = self.get_test_ips_for_host(host.name) + #print host, ips + self.assertTrue(ips) + all_ips.update(ips) + ips = self.get_test_ips_for_host('invalid_host_name') + self.assertFalse(ips) + for ip in all_ips: + host = self.get_test_host_for_ip(ip) + #print ip, host + if ip.startswith('127.30.'): + continue + self.assertTrue(host) + ips = self.get_test_ips_for_host(host) + self.assertTrue(ip in ips) + host = self.get_test_host_for_ip('127.10.254.254') + self.assertFalse(host) + + def gethostbyaddr(self, ip): + #print 'gethostbyaddr', ip + if not ip.startswith('127.'): + return self._original_gethostbyaddr(ip) + host = self.get_test_host_for_ip(ip) + if not host: + raise socket.herror('unknown test host') + raddr = '.'.join(list(reversed(ip.split('.'))) + ['in-addr', 'arpa']) + return (host, [raddr], [ip]) + + def getaddrinfo(self, host, port, family=0, socktype=0, proto=0, flags=0): + #print 'getaddrinfo', host, port, family, socktype, proto, flags + if family or socktype or proto or flags: + return self._original_getaddrinfo(host, port, family, socktype, + proto, flags) + port = port or 0 + try: + socket.inet_aton(host) + addrs = [host] + except socket.error: + addrs = self.get_test_ips_for_host(host) + if not addrs: + raise socket.gaierror('test host not found') + results = [] + for addr in addrs: + results.append((socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_TCP, '', (addr, port))) + results.append((socket.AF_INET, socket.SOCK_DGRAM, + socket.IPPROTO_UDP, '', (addr, port))) + return results + + def test_job_template_callback(self): + # Find a valid job template to use to test the callback. + job_template = None + qs = JobTemplate.objects.filter(job_type='run', + credential__isnull=False) + qs = qs.exclude(host_config_key='') + for jt in qs: + if not jt.can_start_without_user_input(): + continue + job_template = jt + break + self.assertTrue(job_template) + url = reverse('main:job_template_callback', args=(job_template.pk,)) + + # Test a POST to start a new job. + with self.current_user(None): + data = dict(host_config_key=job_template.host_config_key) + host = job_template.inventory.hosts.order_by('-pk')[0] + ip = self.get_test_ips_for_host(host.name)[0] + jobs_qs = job_template.jobs.filter(launch_type='callback') + self.assertEqual(jobs_qs.count(), 0) + self.post(url, data, expect=202, remote_addr=ip) + self.assertEqual(jobs_qs.count(), 1) + job = jobs_qs[0] + self.assertEqual(job.launch_type, 'callback') + self.assertEqual(job.limit, host.name) + self.assertEqual(job.hosts.count(), 1) + self.assertEqual(job.hosts.all()[0], host) diff --git a/awx/main/views.py b/awx/main/views.py index ae52484558..d40e3d76bb 100644 --- a/awx/main/views.py +++ b/awx/main/views.py @@ -4,6 +4,7 @@ # Python import datetime import re +import socket import sys # Django @@ -1070,27 +1071,107 @@ class JobTemplateCallback(generics.RetrieveAPIView): ''' Configure a host to POST to this resource using the `host_config_key`. ''' - + model = JobTemplate permission_classes = (JobTemplateCallbackPermission,) - def get(self, request, *args, **kwargs): + def find_host(self): + ''' + Find the host in the job template's inventory that matches the remote + host for the current request. + ''' + # Find the list of remote host names/IPs to check. + remote_hosts = set() + for header in settings.REMOTE_HOST_HEADERS: + value = self.request.META.get(header, '').strip() + if value: + remote_hosts.add(value) + # Add the reverse lookup of IP addresses. + for rh in list(remote_hosts): + try: + result = socket.gethostbyaddr(rh) + except socket.herror: + continue + remote_hosts.add(result[0]) + remote_hosts.update(result[1]) + # Filter out any .arpa results. + for rh in list(remote_hosts): + if rh.endswith('.arpa'): + remote_hosts.remove(rh) + if not remote_hosts: + return + # Find the host objects to search for a match. obj = self.get_object() + qs = obj.inventory.hosts.filter(active=True) + # First try for an exact match on the name. + try: + return qs.get(name__in=remote_hosts) + except (Host.DoesNotExist, Host.MultipleObjectsReturned): + pass + # Next, try matching based on name or ansible_ssh_host variable. + matches = dict() + for host in qs: + ansible_ssh_host = host.variables_dict.get('ansible_ssh_host', '') + if ansible_ssh_host in remote_hosts: + if host not in matches: + matches[host] = 0 + matches[host] += 2 + if host.name != ansible_ssh_host and host.name in remote_hosts: + if host not in matches: + matches[host] = 0 + matches[host] += 1 + if len(matches) == 1: + return matches.keys()[0] + # Try to resolve forward addresses for each host to find a match. + for host in qs: + hostnames = set([host.name]) + ansible_ssh_host = host.variables_dict.get('ansible_ssh_host', '') + if ansible_ssh_host: + hostnames.add(ansible_ssh_host) + for hostname in hostnames: + try: + result = socket.getaddrinfo(hostname, None) + possible_ips = set(x[4][0] for x in result) + possible_ips.discard(hostname) + if possible_ips and possible_ips & remote_hosts: + if host in matches: + matches[host] += 1 + else: + matches[host] = 1 + except socket.gaierror: + pass + # Return the host with the highest match weight (in case of multiple + # matches). + if matches: + return sorted(matches.items(), key=lambda x: x[1])[-1][0] + + def get(self, request, *args, **kwargs): + job_template = self.get_object() data = dict( - host_config_key=obj.host_config_key, + host_config_key=job_template.host_config_key, + matched_host=getattr(self.find_host(), 'name', None), ) + if settings.DEBUG: + d = dict([(k,v) for k,v in request.META.items() + if k.startswith('HTTP_') or k.startswith('REMOTE_')]) + data['request_meta'] = d return Response(data) def post(self, request, *args, **kwargs): - obj = self.get_object() + job_template = self.get_object() # Permission class should have already validated host_config_key. - # FIXME: Find host from request. - limit = obj.limit - # FIXME: Update limit based on host. - job = obj.create_job(limit=limit, launch_type='callback') + host = self.find_host() + if not host: + data = dict(msg='No matching host could be found!') + return Response(data, status=400) + if not job_template.can_start_without_user_input(): + data = dict(msg='Cannot start automatically, user input required!') + return Response(data, status=400) + limit = ':'.join(filter(None, [job_template.limit, host.name])) + job = job_template.create_job(limit=limit, launch_type='callback') result = job.start() if not result: - data = dict(passwords_needed_to_start=job.get_passwords_needed_to_start()) + data = dict(msg='Error starting job!') return Response(data, status=400) else: return Response(status=202) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 2b780671f1..d7452e56a6 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -92,6 +92,11 @@ SECRET_KEY = 'p7z7g1ql4%6+(6nlebb6hdk7sd^&fnjpal308%n%+p^_e6vo1y' # See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts ALLOWED_HOSTS = [] +# HTTP headers and meta keys to search to determine remote host name or IP. Add +# additional items to this list, such as "HTTP_X_FORWARDED_FOR", if behind a +# reverse proxy. +REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] + TEMPLATE_CONTEXT_PROCESSORS += ( 'django.core.context_processors.request', 'awx.ui.context_processors.settings', diff --git a/awx/settings/local_settings.py.example b/awx/settings/local_settings.py.example index 4ef920b675..2b03e6b5d6 100644 --- a/awx/settings/local_settings.py.example +++ b/awx/settings/local_settings.py.example @@ -55,6 +55,11 @@ LANGUAGE_CODE = 'en-us' # the secret key from an environment variable or a file instead. SECRET_KEY = 'p7z7g1ql4%6+(6nlebb6hdk7sd^&fnjpal308%n%+p^_e6vo1y' +# HTTP headers and meta keys to search to determine remote host name or IP. Add +# additional items to this list, such as "HTTP_X_FORWARDED_FOR", if behind a +# reverse proxy. +REMOTE_HOST_HEADERS = ['REMOTE_ADDR', 'REMOTE_HOST'] + # Email address that error messages come from. SERVER_EMAIL = 'root@localhost'