mirror of
https://github.com/ansible/awx.git
synced 2026-06-26 17:08:03 -02:30
Merge remote-tracking branch 'origin/release_3.1.2' into devel
This commit is contained in:
@@ -190,7 +190,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
|
||||
data = {}
|
||||
for field in ('job_type', 'inventory_id', 'limit', 'credential_id',
|
||||
'module_name', 'module_args', 'forks', 'verbosity',
|
||||
'become_enabled'):
|
||||
'extra_vars', 'become_enabled'):
|
||||
data[field] = getattr(self, field)
|
||||
return AdHocCommand.objects.create(**data)
|
||||
|
||||
|
||||
@@ -1277,10 +1277,20 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
|
||||
def get_notification_friendly_name(self):
|
||||
return "Inventory Update"
|
||||
|
||||
def cancel(self):
|
||||
res = super(InventoryUpdate, self).cancel()
|
||||
def _build_job_explanation(self):
|
||||
if not self.job_explanation:
|
||||
return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
|
||||
(self.model_to_str(), self.name, self.id)
|
||||
return None
|
||||
|
||||
def get_dependent_jobs(self):
|
||||
return Job.objects.filter(dependent_jobs__in=[self.id])
|
||||
|
||||
def cancel(self, job_explanation=None):
|
||||
|
||||
res = super(InventoryUpdate, self).cancel(job_explanation=job_explanation)
|
||||
if res:
|
||||
map(lambda x: x.cancel(), Job.objects.filter(dependent_jobs__in=[self.id]))
|
||||
map(lambda x: x.cancel(job_explanation=self._build_job_explanation()), self.get_dependent_jobs())
|
||||
return res
|
||||
|
||||
|
||||
|
||||
@@ -310,9 +310,13 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, Resour
|
||||
elif self.variables_needed_to_start:
|
||||
variables_needed = True
|
||||
prompting_needed = False
|
||||
for value in self._ask_for_vars_dict().values():
|
||||
if value:
|
||||
prompting_needed = True
|
||||
# The behavior of provisioning callback should mimic
|
||||
# that of job template launch, so prompting_needed should
|
||||
# not block a provisioning callback from creating/launching jobs.
|
||||
if callback_extra_vars is None:
|
||||
for value in self._ask_for_vars_dict().values():
|
||||
if value:
|
||||
prompting_needed = True
|
||||
return (not prompting_needed and
|
||||
not self.passwords_needed_to_start and
|
||||
not variables_needed)
|
||||
@@ -633,10 +637,10 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin):
|
||||
Canceling a job also cancels the implicit project update with launch_type
|
||||
run.
|
||||
'''
|
||||
def cancel(self):
|
||||
res = super(Job, self).cancel()
|
||||
def cancel(self, job_explanation=None):
|
||||
res = super(Job, self).cancel(job_explanation=job_explanation)
|
||||
if self.project_update:
|
||||
self.project_update.cancel()
|
||||
self.project_update.cancel(job_explanation=job_explanation)
|
||||
return res
|
||||
|
||||
|
||||
@@ -1139,7 +1143,7 @@ class JobEvent(CreatedModifiedModel):
|
||||
# Save artifact data to parent job (if provided).
|
||||
if artifact_dict:
|
||||
if event_data and isinstance(event_data, dict):
|
||||
# Note: Core has not added support for marking artifacts as
|
||||
# Note: Core has not added support for marking artifacts as
|
||||
# sensitive yet. Going forward, core will not use
|
||||
# _ansible_no_log to denote sensitive set_stats calls.
|
||||
# Instead, they plan to add a flag outside of the traditional
|
||||
|
||||
@@ -1025,7 +1025,7 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
|
||||
if settings.DEBUG:
|
||||
raise
|
||||
|
||||
def cancel(self):
|
||||
def cancel(self, job_explanation=None):
|
||||
if self.can_cancel:
|
||||
if not self.cancel_flag:
|
||||
self.cancel_flag = True
|
||||
@@ -1033,6 +1033,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
|
||||
if self.status in ('pending', 'waiting', 'new'):
|
||||
self.status = 'canceled'
|
||||
cancel_fields.append('status')
|
||||
if job_explanation is not None:
|
||||
self.job_explanation = job_explanation
|
||||
cancel_fields.append('job_explanation')
|
||||
self.save(update_fields=cancel_fields)
|
||||
self.websocket_emit_status("canceled")
|
||||
if settings.BROKER_URL.startswith('amqp://'):
|
||||
|
||||
@@ -90,7 +90,7 @@ def celery_startup(conf=None, **kwargs):
|
||||
@worker_process_init.connect
|
||||
def task_set_logger_pre_run(*args, **kwargs):
|
||||
cache.close()
|
||||
configure_external_logger(settings, async_flag=False, is_startup=False)
|
||||
configure_external_logger(settings, is_startup=False)
|
||||
|
||||
|
||||
def _clear_cache_keys(set_of_keys):
|
||||
@@ -1160,6 +1160,7 @@ class RunProjectUpdate(BaseTask):
|
||||
'''
|
||||
env = super(RunProjectUpdate, self).build_env(project_update, **kwargs)
|
||||
env = self.add_ansible_venv(env)
|
||||
env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False)
|
||||
env['ANSIBLE_ASK_PASS'] = str(False)
|
||||
env['ANSIBLE_ASK_SUDO_PASS'] = str(False)
|
||||
env['DISPLAY'] = '' # Prevent stupid password popup when running tests.
|
||||
|
||||
@@ -339,6 +339,21 @@ def test_list_created_org_credentials(post, get, organization, org_admin, org_me
|
||||
assert response.data['count'] == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize('order_by', ('password', '-password', 'password,pk', '-password,pk'))
|
||||
@pytest.mark.django_db
|
||||
def test_list_cannot_order_by_encrypted_field(post, get, organization, org_admin, order_by):
|
||||
for i, password in enumerate(('abc', 'def', 'xyz')):
|
||||
response = post(reverse('api:credential_list'), {
|
||||
'organization': organization.id,
|
||||
'name': 'C%d' % i,
|
||||
'password': password
|
||||
}, org_admin)
|
||||
|
||||
response = get(reverse('api:credential_list'), org_admin,
|
||||
QUERY_STRING='order_by=%s' % order_by, status=400)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
#
|
||||
# Openstack Credentials
|
||||
#
|
||||
|
||||
@@ -35,6 +35,21 @@ def test_edit_inventory(put, inventory, alice, role_field, expected_status_code)
|
||||
put(reverse('api:inventory_detail', args=(inventory.id,)), data, alice, expect=expected_status_code)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('order_by', ('script', '-script', 'script,pk', '-script,pk'))
|
||||
@pytest.mark.django_db
|
||||
def test_list_cannot_order_by_unsearchable_field(get, organization, alice, order_by):
|
||||
for i, script in enumerate(('#!/bin/a', '#!/bin/b', '#!/bin/c')):
|
||||
custom_script = organization.custom_inventory_scripts.create(
|
||||
name="I%d" % i,
|
||||
script=script
|
||||
)
|
||||
custom_script.admin_role.members.add(alice)
|
||||
|
||||
response = get(reverse('api:inventory_script_list'), alice,
|
||||
QUERY_STRING='order_by=%s' % order_by, status=400)
|
||||
assert response.status_code == 400
|
||||
|
||||
|
||||
@pytest.mark.parametrize("role_field,expected_status_code", [
|
||||
(None, 403),
|
||||
('admin_role', 201),
|
||||
|
||||
@@ -344,3 +344,53 @@ def test_job_launch_unprompted_vars_with_survey(mocker, survey_spec_factory, job
|
||||
|
||||
# Check that the survey variable is accepted and the job variable isn't
|
||||
mock_job.signal_start.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.job_runtime_vars
|
||||
def test_callback_accept_prompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host):
|
||||
job_template = job_template_prompts(True)
|
||||
job_template.host_config_key = "foo"
|
||||
job_template.survey_enabled = True
|
||||
job_template.survey_spec = survey_spec_factory('survey_var')
|
||||
job_template.save()
|
||||
|
||||
with mocker.patch('awx.main.access.BaseAccess.check_license'):
|
||||
mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4})
|
||||
with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
|
||||
with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}):
|
||||
with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]):
|
||||
post(
|
||||
reverse('api:job_template_callback', args=[job_template.pk]),
|
||||
dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"),
|
||||
admin_user, expect=201, format='json')
|
||||
assert JobTemplate.create_unified_job.called
|
||||
assert JobTemplate.create_unified_job.call_args == ({'extra_vars': {'survey_var': 4,
|
||||
'job_launch_var': 3},
|
||||
'launch_type': 'callback',
|
||||
'limit': 'single-host'},)
|
||||
|
||||
mock_job.signal_start.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
@pytest.mark.job_runtime_vars
|
||||
def test_callback_ignore_unprompted_extra_var(mocker, survey_spec_factory, job_template_prompts, post, admin_user, host):
|
||||
job_template = job_template_prompts(False)
|
||||
job_template.host_config_key = "foo"
|
||||
job_template.save()
|
||||
|
||||
with mocker.patch('awx.main.access.BaseAccess.check_license'):
|
||||
mock_job = mocker.MagicMock(spec=Job, id=968, extra_vars={"job_launch_var": 3, "survey_var": 4})
|
||||
with mocker.patch.object(JobTemplate, 'create_unified_job', return_value=mock_job):
|
||||
with mocker.patch('awx.api.serializers.JobSerializer.to_representation', return_value={}):
|
||||
with mocker.patch('awx.api.views.JobTemplateCallback.find_matching_hosts', return_value=[host]):
|
||||
post(
|
||||
reverse('api:job_template_callback', args=[job_template.pk]),
|
||||
dict(extra_vars={"job_launch_var": 3, "survey_var": 4}, host_config_key="foo"),
|
||||
admin_user, expect=201, format='json')
|
||||
assert JobTemplate.create_unified_job.called
|
||||
assert JobTemplate.create_unified_job.call_args == ({'launch_type': 'callback',
|
||||
'limit': 'single-host'},)
|
||||
|
||||
mock_job.signal_start.assert_called_once()
|
||||
|
||||
@@ -92,3 +92,15 @@ def test_expired_licenses():
|
||||
|
||||
assert vdata['compliant'] is False
|
||||
assert vdata['grace_period_remaining'] > 0
|
||||
|
||||
|
||||
@pytest.mark.django_db
|
||||
def test_cloudforms_license(mocker):
|
||||
with mocker.patch('awx.main.task_engine.TaskEnhancer._check_cloudforms_subscription', return_value=True):
|
||||
task_enhancer = TaskEnhancer()
|
||||
vdata = task_enhancer.validate_enhancements()
|
||||
assert vdata['compliant'] is True
|
||||
assert vdata['subscription_name'] == "Red Hat CloudForms License"
|
||||
assert vdata['available_instances'] == 9999999
|
||||
assert vdata['license_type'] == 'enterprise'
|
||||
assert vdata['features']['ha'] is True
|
||||
|
||||
0
awx/main/tests/unit/models/__init__.py
Normal file
0
awx/main/tests/unit/models/__init__.py
Normal file
38
awx/main/tests/unit/models/test_inventory.py
Normal file
38
awx/main/tests/unit/models/test_inventory.py
Normal file
@@ -0,0 +1,38 @@
|
||||
import pytest
|
||||
import mock
|
||||
from awx.main.models import (
|
||||
UnifiedJob,
|
||||
InventoryUpdate,
|
||||
Job,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dependent_job(mocker):
|
||||
j = Job(id=3, name='I_am_a_job')
|
||||
j.cancel = mocker.MagicMock(return_value=True)
|
||||
return [j]
|
||||
|
||||
|
||||
def test_cancel(mocker, dependent_job):
|
||||
with mock.patch.object(UnifiedJob, 'cancel', return_value=True) as parent_cancel:
|
||||
iu = InventoryUpdate()
|
||||
|
||||
iu.get_dependent_jobs = mocker.MagicMock(return_value=dependent_job)
|
||||
iu.save = mocker.MagicMock()
|
||||
build_job_explanation_mock = mocker.MagicMock()
|
||||
iu._build_job_explanation = mocker.MagicMock(return_value=build_job_explanation_mock)
|
||||
|
||||
iu.cancel()
|
||||
|
||||
parent_cancel.assert_called_with(job_explanation=None)
|
||||
dependent_job[0].cancel.assert_called_with(job_explanation=build_job_explanation_mock)
|
||||
|
||||
|
||||
def test__build_job_explanation():
|
||||
iu = InventoryUpdate(id=3, name='I_am_an_Inventory_Update')
|
||||
|
||||
job_explanation = iu._build_job_explanation()
|
||||
|
||||
assert job_explanation == 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % \
|
||||
('inventory_update', 'I_am_an_Inventory_Update', 3)
|
||||
@@ -115,3 +115,16 @@ def test_job_template_survey_mixin_length(job_template_factory):
|
||||
{'type':'password', 'variable':'my_other_variable'}]}
|
||||
kwargs = obj._update_unified_job_kwargs(extra_vars={'my_variable':'$encrypted$'})
|
||||
assert kwargs['extra_vars'] == '{"my_variable": "my_default"}'
|
||||
|
||||
|
||||
def test_job_template_can_start_with_callback_extra_vars_provided(job_template_factory):
|
||||
objects = job_template_factory(
|
||||
'callback_extra_vars_test',
|
||||
organization='org1',
|
||||
inventory='inventory1',
|
||||
credential='cred1',
|
||||
persisted=False,
|
||||
)
|
||||
obj = objects.job_template
|
||||
obj.ask_variables_on_launch = True
|
||||
assert obj.can_start_without_user_input(callback_extra_vars='{"foo": "bar"}') is True
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
import pytest
|
||||
import mock
|
||||
|
||||
from awx.main.models import (
|
||||
@@ -14,3 +15,38 @@ def test_unified_job_workflow_attributes():
|
||||
|
||||
assert job.spawned_by_workflow is True
|
||||
assert job.workflow_job_id == 1
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def unified_job(mocker):
|
||||
mocker.patch.object(UnifiedJob, 'can_cancel', return_value=True)
|
||||
j = UnifiedJob()
|
||||
j.status = 'pending'
|
||||
j.cancel_flag = None
|
||||
j.save = mocker.MagicMock()
|
||||
j.websocket_emit_status = mocker.MagicMock()
|
||||
return j
|
||||
|
||||
|
||||
def test_cancel(unified_job):
|
||||
|
||||
unified_job.cancel()
|
||||
|
||||
assert unified_job.cancel_flag is True
|
||||
assert unified_job.status == 'canceled'
|
||||
assert unified_job.job_explanation == ''
|
||||
# Note: the websocket emit status check is just reflecting the state of the current code.
|
||||
# Some more thought may want to go into only emitting canceled if/when the job record
|
||||
# status is changed to canceled. Unlike, currently, where it's emitted unconditionally.
|
||||
unified_job.websocket_emit_status.assert_called_with("canceled")
|
||||
unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status'])
|
||||
|
||||
|
||||
def test_cancel_job_explanation(unified_job):
|
||||
job_explanation = 'giggity giggity'
|
||||
|
||||
unified_job.cancel(job_explanation=job_explanation)
|
||||
|
||||
assert unified_job.job_explanation == job_explanation
|
||||
unified_job.save.assert_called_with(update_fields=['cancel_flag', 'status', 'job_explanation'])
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
import base64
|
||||
import cStringIO
|
||||
import json
|
||||
import logging
|
||||
from uuid import uuid4
|
||||
|
||||
from django.conf import settings
|
||||
from django.conf import LazySettings
|
||||
import pytest
|
||||
import requests
|
||||
@@ -44,17 +46,27 @@ def http_adapter():
|
||||
return FakeHTTPAdapter()
|
||||
|
||||
|
||||
def test_https_logging_handler_requests_sync_implementation():
|
||||
handler = HTTPSHandler(async=False)
|
||||
assert not isinstance(handler.session, FuturesSession)
|
||||
assert isinstance(handler.session, requests.Session)
|
||||
@pytest.fixture()
|
||||
def connection_error_adapter():
|
||||
class ConnectionErrorAdapter(requests.adapters.HTTPAdapter):
|
||||
|
||||
def send(self, request, **kwargs):
|
||||
err = requests.packages.urllib3.exceptions.SSLError()
|
||||
raise requests.exceptions.ConnectionError(err, request=request)
|
||||
|
||||
return ConnectionErrorAdapter()
|
||||
|
||||
|
||||
def test_https_logging_handler_requests_async_implementation():
|
||||
handler = HTTPSHandler(async=True)
|
||||
handler = HTTPSHandler()
|
||||
assert isinstance(handler.session, FuturesSession)
|
||||
|
||||
|
||||
def test_https_logging_handler_has_default_http_timeout():
|
||||
handler = HTTPSHandler.from_django_settings(settings)
|
||||
assert handler.http_timeout == 5
|
||||
|
||||
|
||||
@pytest.mark.parametrize('param', PARAM_NAMES.keys())
|
||||
def test_https_logging_handler_defaults(param):
|
||||
handler = HTTPSHandler()
|
||||
@@ -154,18 +166,39 @@ def test_https_logging_handler_skip_log(params, logger_name, expected):
|
||||
assert handler.skip_log(logger_name) is expected
|
||||
|
||||
|
||||
@pytest.mark.parametrize('message_type, async', [
|
||||
('logstash', False),
|
||||
('logstash', True),
|
||||
('splunk', False),
|
||||
('splunk', True),
|
||||
])
|
||||
def test_https_logging_handler_connection_error(connection_error_adapter,
|
||||
dummy_log_record):
|
||||
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
|
||||
message_type='logstash',
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
|
||||
handler.setFormatter(LogstashFormatter())
|
||||
handler.session.mount('http://', connection_error_adapter)
|
||||
|
||||
buff = cStringIO.StringIO()
|
||||
logging.getLogger('awx.main.utils.handlers').addHandler(
|
||||
logging.StreamHandler(buff)
|
||||
)
|
||||
|
||||
async_futures = handler.emit(dummy_log_record)
|
||||
with pytest.raises(requests.exceptions.ConnectionError):
|
||||
[future.result() for future in async_futures]
|
||||
assert 'failed to emit log to external aggregator\nTraceback' in buff.getvalue()
|
||||
|
||||
# we should only log failures *periodically*, so causing *another*
|
||||
# immediate failure shouldn't report a second ConnectionError
|
||||
buff.truncate(0)
|
||||
async_futures = handler.emit(dummy_log_record)
|
||||
with pytest.raises(requests.exceptions.ConnectionError):
|
||||
[future.result() for future in async_futures]
|
||||
assert buff.getvalue() == ''
|
||||
|
||||
|
||||
@pytest.mark.parametrize('message_type', ['logstash', 'splunk'])
|
||||
def test_https_logging_handler_emit(http_adapter, dummy_log_record,
|
||||
message_type, async):
|
||||
message_type):
|
||||
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
|
||||
message_type=message_type,
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
|
||||
async=async)
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
|
||||
handler.setFormatter(LogstashFormatter())
|
||||
handler.session.mount('http://', http_adapter)
|
||||
async_futures = handler.emit(dummy_log_record)
|
||||
@@ -191,14 +224,12 @@ def test_https_logging_handler_emit(http_adapter, dummy_log_record,
|
||||
assert body['message'] == 'User joe logged in'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('async', (True, False))
|
||||
def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
|
||||
dummy_log_record, async):
|
||||
dummy_log_record):
|
||||
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
|
||||
username='user', password='pass',
|
||||
message_type='logstash',
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
|
||||
async=async)
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
|
||||
handler.setFormatter(LogstashFormatter())
|
||||
handler.session.mount('http://', http_adapter)
|
||||
async_futures = handler.emit(dummy_log_record)
|
||||
@@ -209,13 +240,11 @@ def test_https_logging_handler_emit_logstash_with_creds(http_adapter,
|
||||
assert request.headers['Authorization'] == 'Basic %s' % base64.b64encode("user:pass")
|
||||
|
||||
|
||||
@pytest.mark.parametrize('async', (True, False))
|
||||
def test_https_logging_handler_emit_splunk_with_creds(http_adapter,
|
||||
dummy_log_record, async):
|
||||
dummy_log_record):
|
||||
handler = HTTPSHandler(host='127.0.0.1', enabled_flag=True,
|
||||
password='pass', message_type='splunk',
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'],
|
||||
async=async)
|
||||
enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
|
||||
handler.setFormatter(LogstashFormatter())
|
||||
handler.session.mount('http://', http_adapter)
|
||||
async_futures = handler.emit(dummy_log_record)
|
||||
|
||||
@@ -5,8 +5,10 @@
|
||||
import logging
|
||||
import json
|
||||
import requests
|
||||
from requests.exceptions import RequestException
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from copy import copy
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
# loggly
|
||||
import traceback
|
||||
@@ -19,6 +21,8 @@ from awx.main.utils.formatters import LogstashFormatter
|
||||
|
||||
__all__ = ['HTTPSNullHandler', 'BaseHTTPSHandler', 'configure_external_logger']
|
||||
|
||||
logger = logging.getLogger('awx.main.utils.handlers')
|
||||
|
||||
# AWX external logging handler, generally designed to be used
|
||||
# with the accompanying LogstashHandler, derives from python-logstash library
|
||||
# Non-blocking request accomplished by FuturesSession, similar
|
||||
@@ -34,6 +38,7 @@ PARAM_NAMES = {
|
||||
'enabled_loggers': 'LOG_AGGREGATOR_LOGGERS',
|
||||
'indv_facts': 'LOG_AGGREGATOR_INDIVIDUAL_FACTS',
|
||||
'enabled_flag': 'LOG_AGGREGATOR_ENABLED',
|
||||
'http_timeout': 'LOG_AGGREGATOR_HTTP_TIMEOUT',
|
||||
}
|
||||
|
||||
|
||||
@@ -52,17 +57,41 @@ class HTTPSNullHandler(logging.NullHandler):
|
||||
return super(HTTPSNullHandler, self).__init__()
|
||||
|
||||
|
||||
class VerboseThreadPoolExecutor(ThreadPoolExecutor):
|
||||
|
||||
last_log_emit = 0
|
||||
|
||||
def submit(self, func, *args, **kwargs):
|
||||
def _wrapped(*args, **kwargs):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except Exception:
|
||||
# If an exception occurs in a concurrent thread worker (like
|
||||
# a ConnectionError or a read timeout), periodically log
|
||||
# that failure.
|
||||
#
|
||||
# This approach isn't really thread-safe, so we could
|
||||
# potentially log once per thread every 10 seconds, but it
|
||||
# beats logging *every* failed HTTP request in a scenario where
|
||||
# you've typo'd your log aggregator hostname.
|
||||
now = time.time()
|
||||
if now - self.last_log_emit > 10:
|
||||
logger.exception('failed to emit log to external aggregator')
|
||||
self.last_log_emit = now
|
||||
raise
|
||||
return super(VerboseThreadPoolExecutor, self).submit(_wrapped, *args,
|
||||
**kwargs)
|
||||
|
||||
|
||||
class BaseHTTPSHandler(logging.Handler):
|
||||
def __init__(self, fqdn=False, **kwargs):
|
||||
super(BaseHTTPSHandler, self).__init__()
|
||||
self.fqdn = fqdn
|
||||
self.async = kwargs.get('async', True)
|
||||
for fd in PARAM_NAMES:
|
||||
setattr(self, fd, kwargs.get(fd, None))
|
||||
if self.async:
|
||||
self.session = FuturesSession()
|
||||
else:
|
||||
self.session = requests.Session()
|
||||
self.session = FuturesSession(executor=VerboseThreadPoolExecutor(
|
||||
max_workers=2 # this is the default used by requests_futures
|
||||
))
|
||||
self.add_auth_information()
|
||||
|
||||
@classmethod
|
||||
@@ -135,10 +164,8 @@ class BaseHTTPSHandler(logging.Handler):
|
||||
payload_str = json.dumps(payload_input)
|
||||
else:
|
||||
payload_str = payload_input
|
||||
if self.async:
|
||||
return dict(data=payload_str, background_callback=unused_callback)
|
||||
else:
|
||||
return dict(data=payload_str)
|
||||
return dict(data=payload_str, background_callback=unused_callback,
|
||||
timeout=self.http_timeout)
|
||||
|
||||
def skip_log(self, logger_name):
|
||||
if self.host == '' or (not self.enabled_flag):
|
||||
@@ -153,10 +180,6 @@ class BaseHTTPSHandler(logging.Handler):
|
||||
Emit a log record. Returns a list of zero or more
|
||||
``concurrent.futures.Future`` objects.
|
||||
|
||||
When ``self.async`` is True, the list will contain one
|
||||
Future object for each HTTP request made. When ``self.async`` is
|
||||
False, the list will be empty.
|
||||
|
||||
See:
|
||||
https://docs.python.org/3/library/concurrent.futures.html#future-objects
|
||||
http://pythonhosted.org/futures/
|
||||
@@ -177,17 +200,10 @@ class BaseHTTPSHandler(logging.Handler):
|
||||
for key in facts_dict:
|
||||
fact_payload = copy(payload_data)
|
||||
fact_payload.update(facts_dict[key])
|
||||
if self.async:
|
||||
async_futures.append(self._send(fact_payload))
|
||||
else:
|
||||
self._send(fact_payload)
|
||||
async_futures.append(self._send(fact_payload))
|
||||
return async_futures
|
||||
|
||||
if self.async:
|
||||
return [self._send(payload)]
|
||||
|
||||
self._send(payload)
|
||||
return []
|
||||
return [self._send(payload)]
|
||||
except (KeyboardInterrupt, SystemExit):
|
||||
raise
|
||||
except:
|
||||
@@ -209,7 +225,7 @@ def add_or_remove_logger(address, instance):
|
||||
specific_logger.handlers.append(instance)
|
||||
|
||||
|
||||
def configure_external_logger(settings_module, async_flag=True, is_startup=True):
|
||||
def configure_external_logger(settings_module, is_startup=True):
|
||||
|
||||
is_enabled = settings_module.LOG_AGGREGATOR_ENABLED
|
||||
if is_startup and (not is_enabled):
|
||||
@@ -218,7 +234,7 @@ def configure_external_logger(settings_module, async_flag=True, is_startup=True)
|
||||
|
||||
instance = None
|
||||
if is_enabled:
|
||||
instance = BaseHTTPSHandler.from_django_settings(settings_module, async=async_flag)
|
||||
instance = BaseHTTPSHandler.from_django_settings(settings_module)
|
||||
instance.setFormatter(LogstashFormatter(settings_module=settings_module))
|
||||
awx_logger_instance = instance
|
||||
if is_enabled and 'awx' not in settings_module.LOG_AGGREGATOR_LOGGERS:
|
||||
|
||||
Reference in New Issue
Block a user