Merge remote-tracking branch 'origin/release_3.1.2' into devel

This commit is contained in:
Ryan Petrello
2017-03-21 10:39:16 -04:00
74 changed files with 872 additions and 258 deletions

View File

@@ -190,7 +190,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
data = {}
for field in ('job_type', 'inventory_id', 'limit', 'credential_id',
'module_name', 'module_args', 'forks', 'verbosity',
'become_enabled'):
'extra_vars', 'become_enabled'):
data[field] = getattr(self, field)
return AdHocCommand.objects.create(**data)

View File

@@ -1277,10 +1277,20 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
def get_notification_friendly_name(self):
return "Inventory Update"
def cancel(self):
res = super(InventoryUpdate, self).cancel()
def _build_job_explanation(self):
if not self.job_explanation:
return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
(self.model_to_str(), self.name, self.id)
return None
def get_dependent_jobs(self):
return Job.objects.filter(dependent_jobs__in=[self.id])
def cancel(self, job_explanation=None):
res = super(InventoryUpdate, self).cancel(job_explanation=job_explanation)
if res:
map(lambda x: x.cancel(), Job.objects.filter(dependent_jobs__in=[self.id]))
map(lambda x: x.cancel(job_explanation=self._build_job_explanation()), self.get_dependent_jobs())
return res

View File

@@ -310,9 +310,13 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
elif self.variables_needed_to_start:
variables_needed = True
prompting_needed = False
for value in self._ask_for_vars_dict().values():
if value:
prompting_needed = True
# The behavior of provisioning callback should mimic
# that of job template launch, so prompting_needed should
# not block a provisioning callback from creating/launching jobs.
if callback_extra_vars is None:
for value in self._ask_for_vars_dict().values():
if value:
prompting_needed = True
return (not prompting_needed and
not self.passwords_needed_to_start and
not variables_needed)
@@ -633,10 +637,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin):
Canceling a job also cancels the implicit project update with launch_type
run.
'''
def cancel(self):
res = super(Job, self).cancel()
def cancel(self, job_explanation=None):
res = super(Job, self).cancel(job_explanation=job_explanation)
if self.project_update:
self.project_update.cancel()
self.project_update.cancel(job_explanation=job_explanation)
return res
@@ -1139,7 +1143,7 @@ class JobEvent(CreatedModifiedModel):
# Save artifact data to parent job (if provided).
if artifact_dict:
if event_data and isinstance(event_data, dict):
# Note: Core has not added support for marking artifacts as
# Note: Core has not added support for marking artifacts as
# sensitive yet. Going forward, core will not use
# _ansible_no_log to denote sensitive set_stats calls.
# Instead, they plan to add a flag outside of the traditional

View File

@@ -1025,7 +1025,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
if settings.DEBUG:
raise
def cancel(self):
def cancel(self, job_explanation=None):
if self.can_cancel:
if not self.cancel_flag:
self.cancel_flag = True
@@ -1033,6 +1033,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
if self.status in ('pending', 'waiting', 'new'):
self.status = 'canceled'
cancel_fields.append('status')
if job_explanation is not None:
self.job_explanation = job_explanation
cancel_fields.append('job_explanation')
self.save(update_fields=cancel_fields)
self.websocket_emit_status("canceled")
if settings.BROKER_URL.startswith('amqp://'):

View File

@@ -90,7 +90,7 @@ def celery_startup(conf=None, **kwargs):
@worker_process_init.connect
def task_set_logger_pre_run(*args, **kwargs):
cache.close()
configure_external_logger(settings, async_flag=False, is_startup=False)
configure_external_logger(settings, is_startup=False)
def _clear_cache_keys(set_of_keys):
@@ -1160,6 +1160,7 @@ class RunProjectUpdate(BaseTask):
'''
env = super(RunProjectUpdate, self).build_env(project_update, **kwargs)
env = self.add_ansible_venv(env)
env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False)
env['ANSIBLE_ASK_PASS'] = str(False)
env['ANSIBLE_ASK_SUDO_PASS'] = str(False)
env['DISPLAY'] = '' # Prevent stupid password popup when running tests.

View File

@@ -339,6 +339,21 @@ def test_list_created_org_credentials(post, get, organization, org_admin, org_me
assert response.data['count'] == 0
@pytest.mark.parametrize('order_by', ('password', '-password', 'password,pk', '-password,pk'))
@pytest.mark.django_db
def test_list_cannot_order_by_encrypted_field(post, get, organization, org_admin, order_by):
for i, password in enumerate(('abc', 'def', 'xyz')):
response = post(reverse('api:credential_list'), {
'organization': organization.id,
'name': 'C%d' % i,
'password': password
}, org_admin)
response = get(reverse('api:credential_list'), org_admin,
QUERY_STRING='order_by=%s' % order_by, status=400)
assert response.status_code == 400
#
# Openstack Credentials
#

View File

@@ -35,6 +35,21 @@ def test_edit_inventory(put, inventory, alice, role_field, expected_status_code)
put(reverse('api:inventory_detail', args=(inventory.id,)), data, alice, expect=expected_status_code)
@pytest.mark.parametrize('order_by', ('script', '-script', 'script,pk', '-script,pk'))
@pytest.mark.django_db
def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by):
for i, script in enumerate(('#!/bin/a', '#!/bin/b', '#!/bin/c')):
custom_script = organization.custom_inventory_scripts.create(
name="I%d" % i,
script=script
)
custom_script.admin_role.members.add(alice)
response = get(reverse('api:inventory_script_list'), alice,
QUERY_STRING='order_by=%s' % order_by, status=400)
assert response.status_code == 400
@pytest.mark.parametrize("role_field,expected_status_code", [
(None, 403),
('admin_role', 201),

View File

@@ -344,3 +344,53 @@ def test_job_launch_unprompted_vars_with_survey(mocker, survey_spec_factory, job
# Check that the survey variable is accepted and the job variable isn't
mock_job.signal_start.assert_called_once()
@pytest.mark.django_db
@pytest.mark.job_runtime_vars
def test_callback_accept_prompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host):
job_template = job_template_prompts(True)
job_template.host_config_key = "foo"
job_template.survey_enabled = True
job_template.survey_spec = survey_spec_factory('survey_var')
job_template.save()
with mocker.patch('awx.main.access.BaseAccess.check_license'):
mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4})
with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}):
with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]):
post(
reverse('api:job_template_callback', args=[job_template.pk]),
dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"),
admin_user, expect=201, format='json')
assert JobTemplate.create_unified_job.called
assert JobTemplate.create_unified_job.call_args == ({'extra_vars': {'survey_var': 4,
'job_launch_var': 3},
'launch_type': 'callback',
'limit': 'single-host'},)
mock_job.signal_start.assert_called_once()
@pytest.mark.django_db
@pytest.mark.job_runtime_vars
def test_callback_ignore_unprompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host):
job_template = job_template_prompts(False)
job_template.host_config_key = "foo"
job_template.save()
with mocker.patch('awx.main.access.BaseAccess.check_license'):
mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4})
with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}):
with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]):
post(
reverse('api:job_template_callback', args=[job_template.pk]),
dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"),
admin_user, expect=201, format='json')
assert JobTemplate.create_unified_job.called
assert JobTemplate.create_unified_job.call_args == ({'launch_type': 'callback',
'limit': 'single-host'},)
mock_job.signal_start.assert_called_once()

View File

@@ -92,3 +92,15 @@ def test_expired_licenses():
assert vdata['compliant'] is False
assert vdata['grace_period_remaining'] > 0
@pytest.mark.django_db
def test_cloudforms_license(mocker):
with mocker.patch('awx.main.task_engine.TaskEnhancer._check_cloudforms_subscription', return_value=True):
task_enhancer = TaskEnhancer()
vdata = task_enhancer.validate_enhancements()
assert vdata['compliant'] is True
assert vdata['subscription_name'] == "Red Hat CloudForms License"
assert vdata['available_instances'] == 9999999
assert vdata['license_type'] == 'enterprise'
assert vdata['features']['ha'] is True

View File

View File

@@ -0,0 +1,38 @@
import pytest
import mock
from awx.main.models import (
UnifiedJob,
InventoryUpdate,
Job,
)
@pytest.fixture
def dependent_job(mocker):
j = Job(id=3, name='I_am_a_job')
j.cancel = mocker.MagicMock(return_value=True)
return [j]
def test_cancel(mocker, dependent_job):
with mock.patch.object(UnifiedJob, 'cancel', return_value=True) as parent_cancel:
iu = InventoryUpdate()
iu.get_dependent_jobs = mocker.MagicMock(return_value=dependent_job)
iu.save = mocker.MagicMock()
build_job_explanation_mock = mocker.MagicMock()
iu._build_job_explanation = mocker.MagicMock(return_value=build_job_explanation_mock)
iu.cancel()
parent_cancel.assert_called_with(job_explanation=None)
dependent_job[0].cancel.assert_called_with(job_explanation=build_job_explanation_mock)
def test__build_job_explanation():
iu = InventoryUpdate(id=3, name='I_am_an_Inventory_Update')
job_explanation = iu._build_job_explanation()
assert job_explanation == 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
('inventory_update', 'I_am_an_Inventory_Update', 3)

View File

@@ -115,3 +115,16 @@ def test_job_template_survey_mixin_length(job_template_factory):
{'type':'password', 'variable':'my_other_variable'}]}
kwargs = obj._update_unified_job_kwargs(extra_vars={'my_variable':'$encrypted$'})
assert kwargs['extra_vars'] == '{"my_variable": "my_default"}'
def test_job_template_can_start_with_callback_extra_vars_provided(job_template_factory):
objects = job_template_factory(
'callback_extra_vars_test',
organization='org1',
inventory='inventory1',
credential='cred1',
persisted=False,
)
obj = objects.job_template
obj.ask_variables_on_launch = True
assert obj.can_start_without_user_input(callback_extra_vars='{"foo": "bar"}') is True

View File

@@ -1,3 +1,4 @@
import pytest
import mock
from awx.main.models import (
@@ -14,3 +15,38 @@ def test_unified_job_workflow_attributes():
assert job.spawned_by_workflow is True
assert job.workflow_job_id == 1
@pytest.fixture
def unified_job(mocker):
mocker.patch.object(UnifiedJob, 'can_cancel', return_value=True)
j = UnifiedJob()
j.status = 'pending'
j.cancel_flag = None
j.save = mocker.MagicMock()
j.websocket_emit_status = mocker.MagicMock()
return j
def test_cancel(unified_job):
unified_job.cancel()
assert unified_job.cancel_flag is True
assert unified_job.status == 'canceled'
assert unified_job.job_explanation == ''
# Note: the websocket emit status check is just reflecting the state of the current code.
# Some more thought may want to go into only emitting canceled if/when the job record
# status is changed to canceled. Unlike, currently, where it's emitted unconditionally.
unified_job.websocket_emit_status.assert_called_with("canceled")
unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status'])
def test_cancel_job_explanation(unified_job):
job_explanation = 'giggity giggity'
unified_job.cancel(job_explanation=job_explanation)
assert unified_job.job_explanation == job_explanation
unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status', 'job_explanation'])

View File

@@ -1,8 +1,10 @@
import base64
import cStringIO
import json
import logging
from uuid import uuid4
from django.conf import settings
from django.conf import LazySettings
import pytest
import requests
@@ -44,17 +46,27 @@ def http_adapter():
return FakeHTTPAdapter()
def test_https_logging_handler_requests_sync_implementation():
handler = HTTPSHandler(async=False)
assert not isinstance(handler.session, FuturesSession)
assert isinstance(handler.session, requests.Session)
@pytest.fixture()
def connection_error_adapter():
class ConnectionErrorAdapter(requests.adapters.HTTPAdapter):
def send(self, request, **kwargs):
err = requests.packages.urllib3.exceptions.SSLError()
raise requests.exceptions.ConnectionError(err, request=request)
return ConnectionErrorAdapter()
def test_https_logging_handler_requests_async_implementation():
handler = HTTPSHandler(async=True)
handler = HTTPSHandler()
assert isinstance(handler.session, FuturesSession)
def test_https_logging_handler_has_default_http_timeout():
handler = HTTPSHandler.from_django_settings(settings)
assert handler.http_timeout == 5
@pytest.mark.parametrize('param', PARAM_NAMES.keys())
def test_https_logging_handler_defaults(param):
handler = HTTPSHandler()
@@ -154,18 +166,39 @@ def test_https_logging_handler_skip_log(params, logger_name, expected):
assert handler.skip_log(logger_name) is expected
@pytest.mark.parametrize('message_type, async', [
('logstash', False),
('logstash', True),
('splunk', False),
('splunk', True),
])
def test_https_logging_handler_connection_error(connection_error_adapter,
dummy_log_record):
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
message_type='logstash',
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
handler.setFormatter(LogstashFormatter())
handler.session.mount('http://', connection_error_adapter)
buff = cStringIO.StringIO()
logging.getLogger('awx.main.utils.handlers').addHandler(
logging.StreamHandler(buff)
)
async_futures = handler.emit(dummy_log_record)
with pytest.raises(requests.exceptions.ConnectionError):
[future.result() for future in async_futures]
assert 'failed to emit log to external aggregator\nTraceback' in buff.getvalue()
# we should only log failures *periodically*, so causing *another*
# immediate failure shouldn't report a second ConnectionError
buff.truncate(0)
async_futures = handler.emit(dummy_log_record)
with pytest.raises(requests.exceptions.ConnectionError):
[future.result() for future in async_futures]
assert buff.getvalue() == ''
@pytest.mark.parametrize('message_type', ['logstash', 'splunk'])
def test_https_logging_handler_emit(http_adapter, dummy_log_record,
message_type, async):
message_type):
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
message_type=message_type,
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
async=async)
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
handler.setFormatter(LogstashFormatter())
handler.session.mount('http://', http_adapter)
async_futures = handler.emit(dummy_log_record)
@@ -191,14 +224,12 @@ def test_https_logging_handler_emit(http_adapter, dummy_log_record,
assert body['message'] == 'User joe logged in'
@pytest.mark.parametrize('async', (True, False))
def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
dummy_log_record, async):
dummy_log_record):
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
username='user', password='pass',
message_type='logstash',
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
async=async)
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
handler.setFormatter(LogstashFormatter())
handler.session.mount('http://', http_adapter)
async_futures = handler.emit(dummy_log_record)
@@ -209,13 +240,11 @@ def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
assert request.headers['Authorization'] == 'Basic %s' % base64.b64encode("user:pass")
@pytest.mark.parametrize('async', (True, False))
def test_https_logging_handler_emit_splunk_with_creds(http_adapter,
dummy_log_record, async):
dummy_log_record):
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
password='pass', message_type='splunk',
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
async=async)
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
handler.setFormatter(LogstashFormatter())
handler.session.mount('http://', http_adapter)
async_futures = handler.emit(dummy_log_record)

View File

@@ -5,8 +5,10 @@
import logging
import json
import requests
from requests.exceptions import RequestException
import time
from concurrent.futures import ThreadPoolExecutor
from copy import copy
from requests.exceptions import RequestException
# loggly
import traceback
@@ -19,6 +21,8 @@ from awx.main.utils.formatters import LogstashFormatter
__all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'configure_external_logger']
logger = logging.getLogger('awx.main.utils.handlers')
# AWX external logging handler, generally designed to be used
# with the accompanying LogstashHandler, derives from python-logstash library
# Non-blocking request accomplished by FuturesSession, similar
@@ -34,6 +38,7 @@ PARAM_NAMES = {
'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS',
'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS',
'enabled_flag': 'LOG_AGGREGATOR_ENABLED',
'http_timeout': 'LOG_AGGREGATOR_HTTP_TIMEOUT',
}
@@ -52,17 +57,41 @@ class HTTPSNullHandler(logging.NullHandler):
return super(HTTPSNullHandler, self).__init__()
class VerboseThreadPoolExecutor(ThreadPoolExecutor):
last_log_emit = 0
def submit(self, func, *args, **kwargs):
def _wrapped(*args, **kwargs):
try:
return func(*args, **kwargs)
except Exception:
# If an exception occurs in a concurrent thread worker (like
# a ConnectionError or a read timeout), periodically log
# that failure.
#
# This approach isn't really thread-safe, so we could
# potentially log once per thread every 10 seconds, but it
# beats logging *every* failed HTTP request in a scenario where
# you've typo'd your log aggregator hostname.
now = time.time()
if now - self.last_log_emit > 10:
logger.exception('failed to emit log to external aggregator')
self.last_log_emit = now
raise
return super(VerboseThreadPoolExecutor, self).submit(_wrapped, *args,
**kwargs)
class BaseHTTPSHandler(logging.Handler):
def __init__(self, fqdn=False, **kwargs):
super(BaseHTTPSHandler, self).__init__()
self.fqdn = fqdn
self.async = kwargs.get('async', True)
for fd in PARAM_NAMES:
setattr(self, fd, kwargs.get(fd, None))
if self.async:
self.session = FuturesSession()
else:
self.session = requests.Session()
self.session = FuturesSession(executor=VerboseThreadPoolExecutor(
max_workers=2 # this is the default used by requests_futures
))
self.add_auth_information()
@classmethod
@@ -135,10 +164,8 @@ class BaseHTTPSHandler(logging.Handler):
payload_str = json.dumps(payload_input)
else:
payload_str = payload_input
if self.async:
return dict(data=payload_str, background_callback=unused_callback)
else:
return dict(data=payload_str)
return dict(data=payload_str, background_callback=unused_callback,
timeout=self.http_timeout)
def skip_log(self, logger_name):
if self.host == '' or (not self.enabled_flag):
@@ -153,10 +180,6 @@ class BaseHTTPSHandler(logging.Handler):
Emit a log record. Returns a list of zero or more
``concurrent.futures.Future`` objects.
When ``self.async`` is True, the list will contain one
Future object for each HTTP request made. When ``self.async`` is
False, the list will be empty.
See:
https://docs.python.org/3/library/concurrent.futures.html#future-objects
http://pythonhosted.org/futures/
@@ -177,17 +200,10 @@ class BaseHTTPSHandler(logging.Handler):
for key in facts_dict:
fact_payload = copy(payload_data)
fact_payload.update(facts_dict[key])
if self.async:
async_futures.append(self._send(fact_payload))
else:
self._send(fact_payload)
async_futures.append(self._send(fact_payload))
return async_futures
if self.async:
return [self._send(payload)]
self._send(payload)
return []
return [self._send(payload)]
except (KeyboardInterrupt, SystemExit):
raise
except:
@@ -209,7 +225,7 @@ def add_or_remove_logger(address, instance):
specific_logger.handlers.append(instance)
def configure_external_logger(settings_module, async_flag=True, is_startup=True):
def configure_external_logger(settings_module, is_startup=True):
is_enabled = settings_module.LOG_AGGREGATOR_ENABLED
if is_startup and (not is_enabled):
@@ -218,7 +234,7 @@ def configure_external_logger(settings_module, async_flag=True, is_startup=True)
instance = None
if is_enabled:
instance = BaseHTTPSHandler.from_django_settings(settings_module, async=async_flag)
instance = BaseHTTPSHandler.from_django_settings(settings_module)
instance.setFormatter(LogstashFormatter(settings_module=settings_module))
awx_logger_instance = instance
if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS: