External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-21 07:47:44 -02:30
Code Issues Packages Projects Releases Wiki Activity

merging devel

Browse Source
This commit is contained in:
Wayne Witzel III
2016-10-03 10:35:50 -04:00
parent 258ddba333 b4c852cf53
commit 530a5c0c88
110 changed files with 5192 additions and 2396 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
awx/main/__init__.py
View File

@@ -1,2 +1,4 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
default_app_config = 'awx.main.apps.MainConfig'

26
awx/main/access.py
View File

@@ -7,6 +7,7 @@ import sys
import logging
# Django
from django.conf import settings
from django.db.models import Q
from django.contrib.auth.models import User
from django.contrib.contenttypes.models import ContentType
@@ -19,9 +20,8 @@ from awx.main.utils import * # noqa
from awx.main.models import * # noqa
from awx.main.models.unified_jobs import ACTIVE_STATES
from awx.main.models.mixins import ResourceMixin
from awx.api.license import LicenseForbids
from awx.main.task_engine import TaskSerializer
from awx.main.conf import tower_settings
from awx.main.task_engine import TaskEnhancer
from awx.conf.license import LicenseForbids
__all__ = ['get_user_queryset', 'check_user_access',
'user_accessible_objects',
@@ -192,8 +192,7 @@ class BaseAccess(object):
return self.can_change(obj, data)
def check_license(self, add_host=False, feature=None, check_expiration=True):
reader = TaskSerializer()
validation_info = reader.from_database()
validation_info = TaskEnhancer().validate_enhancements()
if ('test' in sys.argv or 'py.test' in sys.argv[0] or 'jenkins' in sys.argv) and not os.environ.get('SKIP_LICENSE_FIXUP_FOR_TEST', ''):
validation_info['free_instances'] = 99999999
validation_info['time_remaining'] = 99999999
@@ -311,7 +310,7 @@ class UserAccess(BaseAccess):
if self.user.is_superuser or self.user.is_system_auditor:
return User.objects.all()
if tower_settings.ORG_ADMINS_CAN_SEE_ALL_USERS and \
if settings.ORG_ADMINS_CAN_SEE_ALL_USERS and \
(self.user.admin_of_organizations.exists() or self.user.auditor_of_organizations.exists()):
return User.objects.all()
@@ -1919,20 +1918,6 @@ class CustomInventoryScriptAccess(BaseAccess):
def can_delete(self, obj):
return self.can_admin(obj)
class TowerSettingsAccess(BaseAccess):
'''
- I can see settings when
- I am a super user
- I can edit settings when
- I am a super user
- I can clear settings when
- I am a super user
'''
model = TowerSettings
class RoleAccess(BaseAccess):
'''
- I can see roles when
@@ -2009,7 +1994,6 @@ register_access(UnifiedJobTemplate, UnifiedJobTemplateAccess)
register_access(UnifiedJob, UnifiedJobAccess)
register_access(ActivityStream, ActivityStreamAccess)
register_access(CustomInventoryScript, CustomInventoryScriptAccess)
register_access(TowerSettings, TowerSettingsAccess)
register_access(Role, RoleAccess)
register_access(NotificationTemplate, NotificationTemplateAccess)
register_access(Notification, NotificationAccess)

9
awx/main/apps.py Normal file
View File

@@ -0,0 +1,9 @@
# Django
from django.apps import AppConfig
from django.utils.translation import ugettext_lazy as _
class MainConfig(AppConfig):
name = 'awx.main'
verbose_name = _('Main')

209
awx/main/conf.py
View File

@@ -1,50 +1,175 @@
# Copyright (c) 2015 Ansible, Inc..
# All Rights Reserved.
# Python
import json
import logging
import os
from django.conf import settings as django_settings
from django.db.utils import ProgrammingError
from django.db import OperationalError
from awx.main.models.configuration import TowerSettings
# Django
from django.utils.translation import ugettext_lazy as _
# Tower
from awx.conf import fields, register
logger = logging.getLogger('awx.main.conf')
class TowerConfiguration(object):
register(
'ACTIVITY_STREAM_ENABLED',
field_class=fields.BooleanField,
label=_('Enable Activity Stream'),
help_text=_('Enable capturing activity for the Tower activity stream.'),
category=_('System'),
category_slug='system',
)
# TODO: Caching so we don't have to hit the database every time for settings
def __getattr__(self, key):
settings_manifest = django_settings.TOWER_SETTINGS_MANIFEST
if key not in settings_manifest:
raise AttributeError("Tower Setting with key '{0}' is not defined in the manifest".format(key))
default_value = settings_manifest[key]['default']
ts = TowerSettings.objects.filter(key=key)
try:
if not ts.exists():
try:
val_actual = getattr(django_settings, key)
except AttributeError:
val_actual = default_value
return val_actual
return ts[0].value_converted
except (ProgrammingError, OperationalError), e:
# Database is not available yet, usually during migrations so lets use the default
logger.debug("Database settings not available yet, using defaults ({0})".format(e))
return default_value
register(
'ACTIVITY_STREAM_ENABLED_FOR_INVENTORY_SYNC',
field_class=fields.BooleanField,
label=_('Enable Activity Stream for Inventory Sync'),
help_text=_('Enable capturing activity for the Tower activity stream when running inventory sync.'),
category=_('System'),
category_slug='system',
)
def __setattr__(self, key, value):
settings_manifest = django_settings.TOWER_SETTINGS_MANIFEST
if key not in settings_manifest:
raise AttributeError("Tower Setting with key '{0}' does not exist".format(key))
settings_entry = settings_manifest[key]
try:
settings_actual = TowerSettings.objects.get(key=key)
except TowerSettings.DoesNotExist:
settings_actual = TowerSettings(key=key,
description=settings_entry['description'],
category=settings_entry['category'],
value_type=settings_entry['type'])
settings_actual.value_converted = value
settings_actual.save()
register(
'ORG_ADMINS_CAN_SEE_ALL_USERS',
field_class=fields.BooleanField,
label=_('All Users Visible to Organization Admins'),
help_text=_('Controls whether any Organization Admin can view all users, even those not associated with their Organization.'),
category=_('System'),
category_slug='system',
)
tower_settings = TowerConfiguration()
register(
'TOWER_ADMIN_ALERTS',
field_class=fields.BooleanField,
label=_('Enable Tower Administrator Alerts'),
help_text=_('Allow Tower to email Admin users for system events that may require attention.'),
category=_('System'),
category_slug='system',
)
register(
'TOWER_URL_BASE',
field_class=fields.URLField,
schemes=('http', 'https'),
allow_plain_hostname=True, # Allow hostname only without TLD.
label=_('Base URL of the Tower host'),
help_text=_('This setting is used by services like notifications to render '
'a valid url to the Tower host.'),
category=_('System'),
category_slug='system',
)
register(
'REMOTE_HOST_HEADERS',
field_class=fields.StringListField,
label=_('Remote Host Headers'),
help_text=_('HTTP headers and meta keys to search to determine remote host '
'name or IP. Add additional items to this list, such as '
'"HTTP_X_FORWARDED_FOR", if behind a reverse proxy.\n\n'
'Note: The headers will be searched in order and the first '
'found remote host name or IP will be used.\n\n'
'In the below example 8.8.8.7 would be the chosen IP address.\n'
'X-Forwarded-For: 8.8.8.7, 192.168.2.1, 127.0.0.1\n'
'Host: 127.0.0.1\n'
'REMOTE_HOST_HEADERS = [\'HTTP_X_FORWARDED_FOR\', '
'\'REMOTE_ADDR\', \'REMOTE_HOST\']'),
category=_('System'),
category_slug='system',
)
def _load_default_license_from_file():
try:
license_file = os.environ.get('AWX_LICENSE_FILE', '/etc/tower/license')
if os.path.exists(license_file):
license_data = json.load(open(license_file))
logger.debug('Read license data from "%s".', license_file)
return license_data
except:
logger.warning('Could not read license from "%s".', license_file, exc_info=True)
return {}
register(
'LICENSE',
field_class=fields.DictField,
default=_load_default_license_from_file,
label=_('Tower License'),
help_text=_('The license controls which features and functionality are '
'enabled in Tower. Use /api/v1/config/ to update or change '
'the license.'),
category=_('System'),
category_slug='system',
)
register(
'AD_HOC_COMMANDS',
field_class=fields.StringListField,
label=_('Ansible Modules Allowed for Ad Hoc Jobs'),
help_text=_('List of modules allowed to be used by ad-hoc jobs.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_PROOT_ENABLED',
field_class=fields.BooleanField,
label=_('Enable PRoot for Job Execution'),
help_text=_('Isolates an Ansible job from protected parts of the Tower system to prevent exposing sensitive information.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_PROOT_BASE_PATH',
field_class=fields.CharField,
label=_('Base PRoot execution path'),
help_text=_('The location that PRoot will create its temporary working directory.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_PROOT_HIDE_PATHS',
field_class=fields.StringListField,
label=_('Paths to hide from PRoot jobs'),
help_text=_('Extra paths to hide from PRoot isolated processes.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_PROOT_SHOW_PATHS',
field_class=fields.StringListField,
label=_('Paths to expose to PRoot jobs'),
help_text=_('Explicit whitelist of paths to expose to PRoot jobs.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'STDOUT_MAX_BYTES_DISPLAY',
field_class=fields.IntegerField,
min_value=0,
label=_('Standard Output Maximum Display Size'),
help_text=_('Maximum Size of Standard Output in bytes to display before requiring the output be downloaded.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'SCHEDULE_MAX_JOBS',
field_class=fields.IntegerField,
min_value=1,
label=_('Maximum Scheduled Jobs'),
help_text=_('Maximum number of the same job template that can be waiting to run when launching from a schedule before no more are created.'),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_ANSIBLE_CALLBACK_PLUGINS',
field_class=fields.StringListField,
label=_('Ansible Callback Plugins'),
help_text=_('List of paths for extra callback plugins to be used when running jobs.'),
category=_('Jobs'),
category_slug='jobs',
)

2
awx/main/management/commands/cleanup_facts.py
View File

@@ -13,7 +13,7 @@ from django.utils.timezone import now
# AWX
from awx.main.models.fact import Fact
from awx.api.license import feature_enabled
from awx.conf.license import feature_enabled
OLDER_THAN = 'older_than'
GRANULARITY = 'granularity'

8
awx/main/management/commands/inventory_import.py
View File

@@ -26,10 +26,9 @@ from django.utils.encoding import smart_text
# AWX
from awx.main.models import * # noqa
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import ignore_inventory_computed_fields, check_proot_installed, wrap_args_with_proot
from awx.main.signals import disable_activity_stream
from awx.main.task_engine import TaskSerializer as LicenseReader
from awx.main.conf import tower_settings
logger = logging.getLogger('awx.main.commands.inventory_import')
@@ -358,7 +357,7 @@ class ExecutableJsonLoader(BaseLoader):
data = {}
stdout, stderr = '', ''
try:
if self.is_custom and getattr(tower_settings, 'AWX_PROOT_ENABLED', False):
if self.is_custom and getattr(settings, 'AWX_PROOT_ENABLED', False):
if not check_proot_installed():
raise RuntimeError("proot is not installed but is configured for use")
kwargs = {'proot_temp_dir': self.source_dir} # TODO: Remove proot dir
@@ -1191,8 +1190,7 @@ class Command(NoArgsCommand):
self._create_update_group_hosts()
def check_license(self):
reader = LicenseReader()
license_info = reader.from_database()
license_info = TaskEnhancer().validate_enhancements()
if not license_info or len(license_info) == 0:
self.logger.error(LICENSE_NON_EXISTANT_MESSAGE)
raise CommandError('No Tower license found!')

2
awx/main/management/commands/run_callback_receiver.py
View File

@@ -117,7 +117,7 @@ class CallbackBrokerWorker(ConsumerMixin):
else:
print("Cache hit")
j.parent_id = parent_id
j.save()
j.save(post_process=True)
if event_uuid:
cache.set("{}_{}".format(payload['job_id'], event_uuid), j.id, 300)
except DatabaseError as e:

483
awx/main/management/commands/run_task_system.py
View File

@@ -1,483 +0,0 @@
#Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
import os
import datetime
import logging
import signal
import time
# Django
from django.conf import settings
from django.core.management.base import NoArgsCommand
# AWX
from awx.main.models import * # noqa
from awx.main.queue import FifoQueue
from awx.main.tasks import handle_work_error, handle_work_success
from awx.main.utils import get_system_task_capacity
# Celery
from celery.task.control import inspect
logger = logging.getLogger('awx.main.commands.run_task_system')
queue = FifoQueue('tower_task_manager')
class SimpleDAG(object):
''' A simple implementation of a directed acyclic graph '''
def __init__(self):
self.nodes = []
self.edges = []
def __contains__(self, obj):
for node in self.nodes:
if node['node_object'] == obj:
return True
return False
def __len__(self):
return len(self.nodes)
def __iter__(self):
return self.nodes.__iter__()
def generate_graphviz_plot(self):
def short_string_obj(obj):
if type(obj) == Job:
type_str = "Job"
if type(obj) == AdHocCommand:
type_str = "AdHocCommand"
elif type(obj) == InventoryUpdate:
type_str = "Inventory"
elif type(obj) == ProjectUpdate:
type_str = "Project"
elif type(obj) == WorkflowJob:
type_str = "Workflow"
else:
type_str = "Unknown"
type_str += "%s" % str(obj.id)
return type_str
doc = """
digraph g {
rankdir = LR
"""
for n in self.nodes:
doc += "%s [color = %s]\n" % (
short_string_obj(n['node_object']),
"red" if n['node_object'].status == 'running' else "black",
)
for from_node, to_node, label in self.edges:
doc += "%s -> %s [ label=\"%s\" ];\n" % (
short_string_obj(self.nodes[from_node]['node_object']),
short_string_obj(self.nodes[to_node]['node_object']),
label,
)
doc += "}\n"
gv_file = open('/tmp/graph.gv', 'w')
gv_file.write(doc)
gv_file.close()
def add_node(self, obj, metadata=None):
if self.find_ord(obj) is None:
self.nodes.append(dict(node_object=obj, metadata=metadata))
def add_edge(self, from_obj, to_obj, label=None):
from_obj_ord = self.find_ord(from_obj)
to_obj_ord = self.find_ord(to_obj)
if from_obj_ord is None or to_obj_ord is None:
raise LookupError("Object not found")
self.edges.append((from_obj_ord, to_obj_ord, label))
def add_edges(self, edgelist):
for edge_pair in edgelist:
self.add_edge(edge_pair[0], edge_pair[1], edge_pair[2])
def find_ord(self, obj):
for idx in range(len(self.nodes)):
if obj == self.nodes[idx]['node_object']:
return idx
return None
def get_node_type(self, obj):
if type(obj) == Job:
return "job"
elif type(obj) == AdHocCommand:
return "ad_hoc_command"
elif type(obj) == InventoryUpdate:
return "inventory_update"
elif type(obj) == ProjectUpdate:
return "project_update"
elif type(obj) == SystemJob:
return "system_job"
elif type(obj) == WorkflowJob:
return "workflow_job"
return "unknown"
def get_dependencies(self, obj, label=None):
antecedents = []
this_ord = self.find_ord(obj)
for node, dep, lbl in self.edges:
if label:
if node == this_ord and lbl == label:
antecedents.append(self.nodes[dep])
else:
if node == this_ord:
antecedents.append(self.nodes[dep])
return antecedents
def get_dependents(self, obj, label=None):
decendents = []
this_ord = self.find_ord(obj)
for node, dep, lbl in self.edges:
if label:
if dep == this_ord and lbl == label:
decendents.append(self.nodes[node])
else:
if dep == this_ord:
decendents.append(self.nodes[node])
return decendents
def get_leaf_nodes(self):
leafs = []
for n in self.nodes:
if len(self.get_dependencies(n['node_object'])) < 1:
leafs.append(n)
return leafs
def get_root_nodes(self):
roots = []
for n in self.nodes:
if len(self.get_dependents(n['node_object'])) < 1:
roots.append(n)
return roots
class WorkflowDAG(SimpleDAG):
def __init__(self, workflow_job=None):
super(WorkflowDAG, self).__init__()
if workflow_job:
self._init_graph(workflow_job)
def _init_graph(self, workflow_job):
workflow_nodes = workflow_job.workflow_job_nodes.all()
for workflow_node in workflow_nodes:
self.add_node(workflow_node)
for node_type in ['success_nodes', 'failure_nodes', 'always_nodes']:
for workflow_node in workflow_nodes:
related_nodes = getattr(workflow_node, node_type).all()
for related_node in related_nodes:
self.add_edge(workflow_node, related_node, node_type)
def bfs_nodes_to_run(self):
root_nodes = self.get_root_nodes()
nodes = root_nodes
nodes_found = []
for index, n in enumerate(nodes):
obj = n['node_object']
job = obj.job
if not job:
nodes_found.append(n)
# Job is about to run or is running. Hold our horses and wait for
# the job to finish. We can't proceed down the graph path until we
# have the job result.
elif job.status not in ['failed', 'error', 'successful']:
continue
elif job.status in ['failed', 'error']:
children_failed = self.get_dependencies(obj, 'failure_nodes')
children_always = self.get_dependencies(obj, 'always_nodes')
children_all = children_failed + children_always
nodes.extend(children_all)
elif job.status in ['successful']:
children_success = self.get_dependencies(obj, 'success_nodes')
nodes.extend(children_success)
else:
logger.warn("Incorrect graph structure")
return [n['node_object'] for n in nodes_found]
def is_workflow_done(self):
root_nodes = self.get_root_nodes()
nodes = root_nodes
for index, n in enumerate(nodes):
obj = n['node_object']
job = obj.job
if not job:
return False
# Job is about to run or is running. Hold our horses and wait for
# the job to finish. We can't proceed down the graph path until we
# have the job result.
elif job.status not in ['failed', 'error', 'successful']:
return False
elif job.status in ['failed', 'error']:
children_failed = self.get_dependencies(obj, 'failure_nodes')
children_always = self.get_dependencies(obj, 'always_nodes')
children_all = children_failed + children_always
nodes.extend(children_all)
elif job.status in ['successful']:
children_success = self.get_dependencies(obj, 'success_nodes')
nodes.extend(children_success)
else:
logger.warn("Incorrect graph structure")
return True
def get_tasks():
"""Fetch all Tower tasks that are relevant to the task management
system.
"""
RELEVANT_JOBS = ('pending', 'waiting', 'running')
# TODO: Replace this when we can grab all objects in a sane way.
graph_jobs = [j for j in Job.objects.filter(status__in=RELEVANT_JOBS)]
graph_ad_hoc_commands = [ahc for ahc in AdHocCommand.objects.filter(status__in=RELEVANT_JOBS)]
graph_inventory_updates = [iu for iu in
InventoryUpdate.objects.filter(status__in=RELEVANT_JOBS)]
graph_project_updates = [pu for pu in
ProjectUpdate.objects.filter(status__in=RELEVANT_JOBS)]
graph_system_jobs = [sj for sj in
SystemJob.objects.filter(status__in=RELEVANT_JOBS)]
graph_workflow_jobs = [wf for wf in
WorkflowJob.objects.filter(status__in=RELEVANT_JOBS)]
all_actions = sorted(graph_jobs + graph_ad_hoc_commands + graph_inventory_updates +
graph_project_updates + graph_system_jobs +
graph_workflow_jobs,
key=lambda task: task.created)
return all_actions
def get_running_workflow_jobs():
graph_workflow_jobs = [wf for wf in
WorkflowJob.objects.filter(status='running')]
return graph_workflow_jobs
def do_spawn_workflow_jobs():
workflow_jobs = get_running_workflow_jobs()
for workflow_job in workflow_jobs:
dag = WorkflowDAG(workflow_job)
spawn_nodes = dag.bfs_nodes_to_run()
for spawn_node in spawn_nodes:
# TODO: Inject job template template params as kwargs.
# Make sure to take into account extra_vars merge logic
kv = {}
job = spawn_node.unified_job_template.create_unified_job(**kv)
spawn_node.job = job
spawn_node.save()
can_start = job.signal_start(**kv)
if not can_start:
job.status = 'failed'
job.job_explanation = "Workflow job could not start because it was not in the right state or required manual credentials"
job.save(update_fields=['status', 'job_explanation'])
job.socketio_emit_status("failed")
# TODO: should we emit a status on the socket here similar to tasks.py tower_periodic_scheduler() ?
#emit_websocket_notification('/socket.io/jobs', '', dict(id=))
def rebuild_graph(message):
"""Regenerate the task graph by refreshing known tasks from Tower, purging
orphaned running tasks, and creating dependencies for new tasks before
generating directed edge relationships between those tasks.
"""
# Sanity check: Only do this on the primary node.
if Instance.objects.my_role() == 'secondary':
return None
inspector = inspect()
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
active_task_queues = inspector.active()
else:
logger.warn("Ignoring celery task inspector")
active_task_queues = None
do_spawn_workflow_jobs()
all_sorted_tasks = get_tasks()
if not len(all_sorted_tasks):
return None
active_tasks = []
if active_task_queues is not None:
for queue in active_task_queues:
active_tasks += [at['id'] for at in active_task_queues[queue]]
else:
logger.error("Could not communicate with celery!")
# TODO: Something needs to be done here to signal to the system
# as a whole that celery appears to be down.
if not hasattr(settings, 'CELERY_UNIT_TEST'):
return None
running_tasks = filter(lambda t: t.status == 'running', all_sorted_tasks)
waiting_tasks = filter(lambda t: t.status != 'running', all_sorted_tasks)
new_tasks = filter(lambda t: t.status == 'pending', all_sorted_tasks)
# Check running tasks and make sure they are active in celery
logger.debug("Active celery tasks: " + str(active_tasks))
for task in list(running_tasks):
if (task.celery_task_id not in active_tasks and not hasattr(settings, 'IGNORE_CELERY_INSPECTOR')):
# NOTE: Pull status again and make sure it didn't finish in
# the meantime?
task.status = 'failed'
task.job_explanation += ' '.join((
'Task was marked as running in Tower but was not present in',
'Celery, so it has been marked as failed.',
))
task.save()
task.websocket_emit_status("failed")
running_tasks.pop(running_tasks.index(task))
logger.error("Task %s appears orphaned... marking as failed" % task)
# Create and process dependencies for new tasks
for task in new_tasks:
logger.debug("Checking dependencies for: %s" % str(task))
try:
task_dependencies = task.generate_dependencies(running_tasks + waiting_tasks)
except Exception, e:
logger.error("Failed processing dependencies for {}: {}".format(task, e))
task.status = 'failed'
task.job_explanation += 'Task failed to generate dependencies: {}'.format(e)
task.save()
task.websocket_emit_status("failed")
continue
logger.debug("New dependencies: %s" % str(task_dependencies))
for dep in task_dependencies:
# We recalculate the created time for the moment to ensure the
# dependencies are always sorted in the right order relative to
# the dependent task.
time_delt = len(task_dependencies) - task_dependencies.index(dep)
dep.created = task.created - datetime.timedelta(seconds=1 + time_delt)
dep.status = 'waiting'
dep.save()
waiting_tasks.insert(waiting_tasks.index(task), dep)
if not hasattr(settings, 'UNIT_TEST_IGNORE_TASK_WAIT'):
task.status = 'waiting'
task.save()
# Rebuild graph
graph = SimpleDAG()
for task in running_tasks:
graph.add_node(task)
for wait_task in waiting_tasks[:50]:
node_dependencies = []
for node in graph:
if wait_task.is_blocked_by(node['node_object']):
node_dependencies.append(node['node_object'])
graph.add_node(wait_task)
for dependency in node_dependencies:
graph.add_edge(wait_task, dependency)
if settings.DEBUG:
graph.generate_graphviz_plot()
return graph
def process_graph(graph, task_capacity):
"""Given a task dependency graph, start and manage tasks given their
priority and weight.
"""
leaf_nodes = graph.get_leaf_nodes()
running_nodes = filter(lambda x: x['node_object'].status == 'running', leaf_nodes)
running_impact = sum([t['node_object'].task_impact for t in running_nodes])
ready_nodes = filter(lambda x: x['node_object'].status != 'running', leaf_nodes)
remaining_volume = task_capacity - running_impact
logger.info('Running Nodes: %s; Capacity: %s; Running Impact: %s; '
'Remaining Capacity: %s' %
(str(running_nodes), str(task_capacity),
str(running_impact), str(remaining_volume)))
logger.info("Ready Nodes: %s" % str(ready_nodes))
for task_node in ready_nodes:
node_obj = task_node['node_object']
# NOTE: This could be used to pass metadata through the task system
# node_args = task_node['metadata']
impact = node_obj.task_impact
if impact <= remaining_volume or running_impact == 0:
node_dependencies = graph.get_dependents(node_obj)
# Allow other tasks to continue if a job fails, even if they are
# other jobs.
if graph.get_node_type(node_obj) == 'job':
node_dependencies = []
dependent_nodes = [{'type': graph.get_node_type(node_obj), 'id': node_obj.id}] + \
[{'type': graph.get_node_type(n['node_object']),
'id': n['node_object'].id} for n in node_dependencies]
error_handler = handle_work_error.s(subtasks=dependent_nodes)
success_handler = handle_work_success.s(task_actual={'type': graph.get_node_type(node_obj),
'id': node_obj.id})
start_status = node_obj.start(error_callback=error_handler, success_callback=success_handler)
if not start_status:
node_obj.status = 'failed'
if node_obj.job_explanation:
node_obj.job_explanation += ' '
node_obj.job_explanation += 'Task failed pre-start check.'
node_obj.save()
continue
remaining_volume -= impact
running_impact += impact
logger.info('Started Node: %s (capacity hit: %s) '
'Remaining Capacity: %s' %
(str(node_obj), str(impact), str(remaining_volume)))
def run_taskmanager():
"""Receive task start and finish signals to rebuild a dependency graph
and manage the actual running of tasks.
"""
def shutdown_handler():
def _handler(signum, frame):
signal.signal(signum, signal.SIG_DFL)
os.kill(os.getpid(), signum)
return _handler
signal.signal(signal.SIGINT, shutdown_handler())
signal.signal(signal.SIGTERM, shutdown_handler())
paused = False
task_capacity = get_system_task_capacity()
last_rebuild = datetime.datetime.fromtimestamp(0)
# Attempt to pull messages off of the task system queue into perpetuity.
#
# A quick explanation of what is happening here:
# The popping messages off the queue bit is something of a sham. We remove
# the messages from the queue and then immediately throw them away. The
# `rebuild_graph` function, while it takes the message as an argument,
# ignores it.
#
# What actually happens is that we just check the database every 10 seconds
# to see what the task dependency graph looks like, and go do that. This
# is the job of the `rebuild_graph` function.
#
# There is some placeholder here: we may choose to actually use the message
# in the future.
while True:
# Pop a message off the queue.
# (If the queue is empty, None will be returned.)
message = queue.pop()
# Parse out the message appropriately, rebuilding our graph if
# appropriate.
if (datetime.datetime.now() - last_rebuild).seconds > 10:
if message is not None and 'pause' in message:
logger.info("Pause command received: %s" % str(message))
paused = message['pause']
graph = rebuild_graph(message)
if not paused and graph is not None:
process_graph(graph, task_capacity)
last_rebuild = datetime.datetime.now()
time.sleep(0.1)
class Command(NoArgsCommand):
"""Tower Task Management System
This daemon is designed to reside between our tasks and celery and
provide a mechanism for understanding the relationship between those tasks
and their dependencies.
It also actively prevents situations in which Tower can get blocked
because it doesn't have an understanding of what is progressing through
celery.
"""
help = 'Launch the Tower task management system'
def handle_noargs(self, **options):
try:
run_taskmanager()
except KeyboardInterrupt:
pass

2
awx/main/management/commands/update_instance.py
View File

@@ -5,7 +5,7 @@ from django.core.management.base import CommandError
from django.db import transaction
from awx.main.management.commands._base_instance import BaseCommandInstance
from awx.api.license import feature_enabled
from awx.conf.license import feature_enabled
from awx.main.models import Instance
instance_str = BaseCommandInstance.instance_str

3
awx/main/managers.py
View File

@@ -4,6 +4,7 @@
import sys
from django.db import models
from django.conf import settings
class HostManager(models.Manager):
@@ -25,7 +26,7 @@ class InstanceManager(models.Manager):
def me(self):
"""Return the currently active instance."""
# If we are running unit tests, return a stub record.
if len(sys.argv) >= 2 and sys.argv[1] == 'test':
if settings.IS_TESTING(sys.argv):
return self.model(id=1,
hostname='localhost',
uuid='00000000-0000-0000-0000-000000000000')

4
awx/main/middleware.py
View File

@@ -5,13 +5,13 @@ import logging
import threading
import uuid
from django.conf import settings
from django.contrib.auth.models import User
from django.db.models.signals import post_save
from django.db import IntegrityError
from django.utils.functional import curry
from awx.main.models import ActivityStream
from awx.main.conf import tower_settings
from awx.api.authentication import TokenAuthentication
@@ -79,6 +79,6 @@ class AuthTokenTimeoutMiddleware(object):
if not TokenAuthentication._get_x_auth_token_header(request):
return response
response['Auth-Token-Timeout'] = int(tower_settings.AUTH_TOKEN_EXPIRATION)
response['Auth-Token-Timeout'] = int(settings.AUTH_TOKEN_EXPIRATION)
return response

22
awx/main/migrations/0036_v310_remove_tower_settings.py Normal file
View File

@@ -0,0 +1,22 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0035_v310_jobevent_uuid'),
]
# These settings are now in the separate awx.conf app.
operations = [
migrations.RemoveField(
model_name='towersettings',
name='user',
),
migrations.DeleteModel(
name='TowerSettings',
),
]

19
awx/main/migrations/0037_v310_job_allow_simultaneous.py Normal file
View File

@@ -0,0 +1,19 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0036_v310_remove_tower_settings'),
]
operations = [
migrations.AddField(
model_name='job',
name='allow_simultaneous',
field=models.BooleanField(default=False),
),
]

35
awx/main/migrations/_old_access.py
View File

@@ -13,6 +13,7 @@ import sys
import logging
# Django
from django.conf import settings
from django.db.models import F, Q
from django.contrib.auth.models import User
@@ -22,9 +23,7 @@ from rest_framework.exceptions import ParseError, PermissionDenied
# AWX
from awx.main.utils import * # noqa
from awx.main.models import * # noqa
from awx.api.license import LicenseForbids
from awx.main.task_engine import TaskSerializer
from awx.main.conf import tower_settings
from awx.conf.license import LicenseForbids
__all__ = ['get_user_queryset', 'check_user_access']
@@ -153,8 +152,8 @@ class BaseAccess(object):
return self.can_change(obj, None)
def check_license(self, add_host=False, feature=None, check_expiration=True):
reader = TaskSerializer()
validation_info = reader.from_database()
from awx.main.task_engine import TaskEnhancer
validation_info = TaskEnhancer().validate_enhancements()
if ('test' in sys.argv or 'py.test' in sys.argv[0] or 'jenkins' in sys.argv) and not os.environ.get('SKIP_LICENSE_FIXUP_FOR_TEST', ''):
validation_info['free_instances'] = 99999999
validation_info['time_remaining'] = 99999999
@@ -202,7 +201,7 @@ class UserAccess(BaseAccess):
qs = self.model.objects.distinct()
if self.user.is_superuser:
return qs
if tower_settings.ORG_ADMINS_CAN_SEE_ALL_USERS and self.user.deprecated_admin_of_organizations.all().exists():
if settings.ORG_ADMINS_CAN_SEE_ALL_USERS and self.user.deprecated_admin_of_organizations.all().exists():
return qs
return qs.filter(
Q(pk=self.user.pk) |
@@ -1624,29 +1623,6 @@ class CustomInventoryScriptAccess(BaseAccess):
return False
class TowerSettingsAccess(BaseAccess):
'''
- I can see settings when
- I am a super user
- I can edit settings when
- I am a super user
- I can clear settings when
- I am a super user
'''
model = TowerSettings
def get_queryset(self):
if self.user.is_superuser:
return self.model.objects.all()
return self.model.objects.none()
def can_change(self, obj, data):
return self.user.is_superuser
def can_delete(self, obj):
return self.user.is_superuser
register_access(User, UserAccess)
register_access(Organization, OrganizationAccess)
register_access(Inventory, InventoryAccess)
@@ -1672,4 +1648,3 @@ register_access(UnifiedJobTemplate, UnifiedJobTemplateAccess)
register_access(UnifiedJob, UnifiedJobAccess)
register_access(ActivityStream, ActivityStreamAccess)
register_access(CustomInventoryScript, CustomInventoryScriptAccess)
register_access(TowerSettings, TowerSettingsAccess)

2
awx/main/models/__init__.py
View File

@@ -16,7 +16,6 @@ from awx.main.models.ad_hoc_commands import * # noqa
from awx.main.models.schedules import * # noqa
from awx.main.models.activity_stream import * # noqa
from awx.main.models.ha import * # noqa
from awx.main.models.configuration import * # noqa
from awx.main.models.rbac import * # noqa
from awx.main.models.mixins import * # noqa
from awx.main.models.notifications import * # noqa
@@ -99,7 +98,6 @@ activity_stream_registrar.connect(AdHocCommand)
# activity_stream_registrar.connect(Profile)
activity_stream_registrar.connect(Schedule)
activity_stream_registrar.connect(CustomInventoryScript)
activity_stream_registrar.connect(TowerSettings)
activity_stream_registrar.connect(NotificationTemplate)
activity_stream_registrar.connect(Notification)
activity_stream_registrar.connect(Label)

5
awx/main/models/ad_hoc_commands.py
View File

@@ -22,7 +22,6 @@ from jsonfield import JSONField
from awx.main.models.base import * # noqa
from awx.main.models.unified_jobs import * # noqa
from awx.main.utils import decrypt_field
from awx.main.conf import tower_settings
from awx.main.models.notifications import JobNotificationMixin
logger = logging.getLogger('awx.main.models.ad_hoc_commands')
@@ -115,7 +114,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
if type(self.module_name) not in (str, unicode):
raise ValidationError("Invalid type for ad hoc command")
module_name = self.module_name.strip() or 'command'
if module_name not in tower_settings.AD_HOC_COMMANDS:
if module_name not in settings.AD_HOC_COMMANDS:
raise ValidationError('Unsupported module for ad hoc commands.')
return module_name
@@ -148,7 +147,7 @@ class AdHocCommand(UnifiedJob, JobNotificationMixin):
return reverse('api:ad_hoc_command_detail', args=(self.pk,))
def get_ui_url(self):
return urljoin(tower_settings.TOWER_URL_BASE, "/#/ad_hoc_commands/{}".format(self.pk))
return urljoin(settings.TOWER_URL_BASE, "/#/ad_hoc_commands/{}".format(self.pk))
@property
def task_auth_token(self):

84
awx/main/models/configuration.py
View File

@@ -1,84 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
# Python
import json
# Django
from django.db import models
from django.utils.encoding import force_text
from django.utils.translation import ugettext_lazy as _
# Tower
from awx.main.models.base import CreatedModifiedModel
class TowerSettings(CreatedModifiedModel):
class Meta:
app_label = 'main'
SETTINGS_TYPE_CHOICES = [
('string', _("String")),
('int', _('Integer')),
('float', _('Decimal')),
('json', _('JSON')),
('bool', _('Boolean')),
('password', _('Password')),
('list', _('List'))
]
key = models.CharField(
max_length=255,
unique=True
)
description = models.TextField()
category = models.CharField(max_length=128)
value = models.TextField(
blank=True,
)
value_type = models.CharField(
max_length=12,
choices=SETTINGS_TYPE_CHOICES
)
user = models.ForeignKey(
'auth.User',
related_name='settings',
default=None,
null=True,
editable=False,
)
@property
def value_converted(self):
if self.value_type == 'json':
converted_type = json.loads(self.value)
elif self.value_type == 'password':
converted_type = self.value
elif self.value_type == 'list':
if self.value:
converted_type = [x.strip() for x in self.value.split(',')]
else:
converted_type = []
elif self.value_type == 'bool':
converted_type = force_text(self.value).lower() in ('true', 'yes', '1')
elif self.value_type == 'string':
converted_type = self.value
else:
t = __builtins__[self.value_type]
converted_type = t(self.value)
return converted_type
@value_converted.setter
def value_converted(self, value):
if self.value_type == 'json':
self.value = json.dumps(value)
elif self.value_type == 'list':
try:
self.value = ','.join(map(force_text, value))
except TypeError:
self.value = force_text(value)
elif self.value_type == 'bool':
self.value = force_text(bool(value))
else:
self.value = force_text(value)

137
awx/main/models/credential.py
View File

@@ -1,9 +1,6 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
import base64
import re
# Django
from django.db import models
from django.utils.translation import ugettext_lazy as _
@@ -14,6 +11,7 @@ from django.core.urlresolvers import reverse
from awx.main.fields import ImplicitRoleField
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.utils import decrypt_field
from awx.main.validators import validate_ssh_private_key
from awx.main.models.base import * # noqa
from awx.main.models.mixins import ResourceMixin
from awx.main.models.rbac import (
@@ -241,11 +239,13 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
else:
ssh_key_data = self.ssh_key_data
try:
key_data = validate_ssh_private_key(ssh_key_data)
pem_objects = validate_ssh_private_key(ssh_key_data)
for pem_object in pem_objects:
if pem_object.get('key_enc', False):
return True
except ValidationError:
return False
else:
return bool(key_data['key_enc'])
pass
return False
@property
def needs_ssh_key_unlock(self):
@@ -379,126 +379,3 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
if 'cloud' not in update_fields:
update_fields.append('cloud')
super(Credential, self).save(*args, **kwargs)
def validate_ssh_private_key(data):
"""Validate that the given SSH private key or certificate is,
in fact, valid.
"""
# Map the X in BEGIN X PRIVATE KEY to the key type (ssh-keygen -t).
# Tower jobs using OPENSSH format private keys may still fail if the
# system SSH implementation lacks support for this format.
key_types = {
'RSA': 'rsa',
'DSA': 'dsa',
'EC': 'ecdsa',
'OPENSSH': 'ed25519',
'': 'rsa1',
}
# Key properties to return if valid.
key_data = {
'key_type': None, # Key type (from above mapping).
'key_seg': '', # Key segment (all text including begin/end).
'key_b64': '', # Key data as base64.
'key_bin': '', # Key data as binary.
'key_enc': None, # Boolean, whether key is encrypted.
'cert_seg': '', # Cert segment (all text including begin/end).
'cert_b64': '', # Cert data as base64.
'cert_bin': '', # Cert data as binary.
}
data = data.strip()
validation_error = ValidationError('Invalid private key.')
# Sanity check: We may potentially receive a full PEM certificate,
# and we want to accept these.
cert_begin_re = r'(-{4,})\s*BEGIN\s+CERTIFICATE\s*(-{4,})'
cert_end_re = r'(-{4,})\s*END\s+CERTIFICATE\s*(-{4,})'
cert_begin_match = re.search(cert_begin_re, data)
cert_end_match = re.search(cert_end_re, data)
if cert_begin_match and not cert_end_match:
raise validation_error
elif not cert_begin_match and cert_end_match:
raise validation_error
elif cert_begin_match and cert_end_match:
cert_dashes = set([cert_begin_match.groups()[0], cert_begin_match.groups()[1],
cert_end_match.groups()[0], cert_end_match.groups()[1]])
if len(cert_dashes) != 1:
raise validation_error
key_data['cert_seg'] = data[cert_begin_match.start():cert_end_match.end()]
# Find the private key, and also ensure that it internally matches
# itself.
# Set up the valid private key header and footer.
begin_re = r'(-{4,})\s*BEGIN\s+([A-Z0-9]+)?\s*PRIVATE\sKEY\s*(-{4,})'
end_re = r'(-{4,})\s*END\s+([A-Z0-9]+)?\s*PRIVATE\sKEY\s*(-{4,})'
begin_match = re.search(begin_re, data)
end_match = re.search(end_re, data)
if not begin_match or not end_match:
raise validation_error
# Ensure that everything, such as dash counts and key type, lines up,
# and raise an error if it does not.
dashes = set([begin_match.groups()[0], begin_match.groups()[2],
end_match.groups()[0], end_match.groups()[2]])
if len(dashes) != 1:
raise validation_error
if begin_match.groups()[1] != end_match.groups()[1]:
raise validation_error
key_type = begin_match.groups()[1] or ''
try:
key_data['key_type'] = key_types[key_type]
except KeyError:
raise ValidationError('Invalid private key: unsupported type %s' % key_type)
# The private key data begins and ends with the private key.
key_data['key_seg'] = data[begin_match.start():end_match.end()]
# Establish that we are able to base64 decode the private key;
# if we can't, then it's not a valid key.
#
# If we got a certificate, validate that also, in the same way.
header_re = re.compile(r'^(.+?):\s*?(.+?)(\\??)$')
for segment_name in ('cert', 'key'):
segment_to_validate = key_data['%s_seg' % segment_name]
# If we have nothing; skip this one.
# We've already validated that we have a private key above,
# so we don't need to do it again.
if not segment_to_validate:
continue
# Ensure that this segment is valid base64 data.
base64_data = ''
line_continues = False
lines = segment_to_validate.splitlines()
for line in lines[1:-1]:
line = line.strip()
if not line:
continue
if line_continues:
line_continues = line.endswith('\\')
continue
line_match = header_re.match(line)
if line_match:
line_continues = line.endswith('\\')
continue
base64_data += line
try:
decoded_data = base64.b64decode(base64_data)
if not decoded_data:
raise validation_error
key_data['%s_b64' % segment_name] = base64_data
key_data['%s_bin' % segment_name] = decoded_data
except TypeError:
raise validation_error
# Determine if key is encrypted.
if key_data['key_type'] == 'ed25519':
# See https://github.com/openssh/openssh-portable/blob/master/sshkey.c#L3218
# Decoded key data starts with magic string (null-terminated), four byte
# length field, followed by the ciphername -- if ciphername is anything
# other than 'none' the key is encrypted.
key_data['key_enc'] = not bool(key_data['key_bin'].startswith('openssh-key-v1\x00\x00\x00\x00\x04none'))
else:
key_data['key_enc'] = bool('ENCRYPTED' in key_data['key_seg'])
return key_data

3
awx/main/models/inventory.py
View File

@@ -30,7 +30,6 @@ from awx.main.models.notifications import (
JobNotificationMixin,
)
from awx.main.utils import _inventory_updates
from awx.main.conf import tower_settings
__all__ = ['Inventory', 'Host', 'Group', 'InventorySource', 'InventoryUpdate', 'CustomInventoryScript']
@@ -1244,7 +1243,7 @@ class InventoryUpdate(UnifiedJob, InventorySourceOptions, JobNotificationMixin):
return reverse('api:inventory_update_detail', args=(self.pk,))
def get_ui_url(self):
return urljoin(tower_settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk))
return urljoin(settings.TOWER_URL_BASE, "/#/inventory_sync/{}".format(self.pk))
def is_blocked_by(self, obj):
if type(obj) == InventoryUpdate:

19
awx/main/models/jobs.py
View File

@@ -30,7 +30,6 @@ from awx.main.models.notifications import (
)
from awx.main.utils import decrypt_field, ignore_inventory_computed_fields
from awx.main.redact import PlainTextCleaner
from awx.main.conf import tower_settings
from awx.main.fields import ImplicitRoleField
from awx.main.models.mixins import ResourceMixin
@@ -140,6 +139,9 @@ class JobOptions(BaseModel):
become_enabled = models.BooleanField(
default=False,
)
allow_simultaneous = models.BooleanField(
default=False,
)
extra_vars_dict = VarsDictProperty('extra_vars', True)
@@ -238,9 +240,6 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, ResourceMixin):
read_role = ImplicitRoleField(
parent_role=['project.organization.auditor_role', 'inventory.organization.auditor_role', 'execute_role', 'admin_role'],
)
allow_simultaneous = models.BooleanField(
default=False,
)
@classmethod
@@ -253,7 +252,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, ResourceMixin):
'playbook', 'credential', 'cloud_credential', 'network_credential', 'forks', 'schedule',
'limit', 'verbosity', 'job_tags', 'extra_vars', 'launch_type',
'force_handlers', 'skip_tags', 'start_at_task', 'become_enabled',
'labels', 'survey_passwords']
'labels', 'survey_passwords', 'allow_simultaneous',]
def resource_validation_data(self):
'''
@@ -484,9 +483,9 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, ResourceMixin):
@property
def cache_timeout_blocked(self):
if Job.objects.filter(job_template=self, status__in=['pending', 'waiting', 'running']).count() > getattr(tower_settings, 'SCHEDULE_MAX_JOBS', 10):
if Job.objects.filter(job_template=self, status__in=['pending', 'waiting', 'running']).count() > getattr(settings, 'SCHEDULE_MAX_JOBS', 10):
logger.error("Job template %s could not be started because there are more than %s other jobs from that template waiting to run" %
(self.name, getattr(tower_settings, 'SCHEDULE_MAX_JOBS', 10)))
(self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10)))
return True
return False
@@ -553,7 +552,7 @@ class Job(UnifiedJob, JobOptions, JobNotificationMixin):
return reverse('api:job_detail', args=(self.pk,))
def get_ui_url(self):
return urljoin(tower_settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk))
return urljoin(settings.TOWER_URL_BASE, "/#/jobs/{}".format(self.pk))
@property
def task_auth_token(self):
@@ -618,7 +617,7 @@ class Job(UnifiedJob, JobOptions, JobNotificationMixin):
if obj.job_template is not None and obj.inventory is not None:
if obj.job_template == self.job_template and \
obj.inventory == self.inventory:
if self.job_template.allow_simultaneous:
if self.allow_simultaneous:
return False
if obj.launch_type == 'callback' and self.launch_type == 'callback' and \
obj.limit != self.limit:
@@ -1376,7 +1375,7 @@ class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):
return reverse('api:system_job_detail', args=(self.pk,))
def get_ui_url(self):
return urljoin(tower_settings.TOWER_URL_BASE, "/#/management_jobs/{}".format(self.pk))
return urljoin(settings.TOWER_URL_BASE, "/#/management_jobs/{}".format(self.pk))
def is_blocked_by(self, obj):
return True

10
awx/main/models/notifications.py
View File

@@ -181,11 +181,11 @@ class JobNotificationMixin(object):
def _build_notification_message(self, status_str):
notification_body = self.notification_data()
notification_subject = "{} #{} '{}' {} on Ansible Tower: {}".format(self.get_notification_friendly_name(),
self.id,
self.name,
status_str,
notification_body['url'])
notification_subject = u"{} #{} '{}' {} on Ansible Tower: {}".format(self.get_notification_friendly_name(),
self.id,
self.name,
status_str,
notification_body['url'])
notification_body['friendly_name'] = self.get_notification_friendly_name()
return (notification_subject, notification_body)

9
awx/main/models/organization.py
View File

@@ -23,7 +23,6 @@ from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_AUDITOR,
)
from awx.main.models.mixins import ResourceMixin
from awx.main.conf import tower_settings
__all__ = ['Organization', 'Team', 'Permission', 'Profile', 'AuthToken']
@@ -262,7 +261,7 @@ class AuthToken(BaseModel):
if not now:
now = tz_now()
if not self.pk or not self.is_expired(now=now):
self.expires = now + datetime.timedelta(seconds=tower_settings.AUTH_TOKEN_EXPIRATION)
self.expires = now + datetime.timedelta(seconds=settings.AUTH_TOKEN_EXPIRATION)
if save:
self.save()
@@ -279,12 +278,12 @@ class AuthToken(BaseModel):
if now is None:
now = tz_now()
invalid_tokens = AuthToken.objects.none()
if tower_settings.AUTH_TOKEN_PER_USER != -1:
if settings.AUTH_TOKEN_PER_USER != -1:
invalid_tokens = AuthToken.objects.filter(
user=user,
expires__gt=now,
reason='',
).order_by('-created')[tower_settings.AUTH_TOKEN_PER_USER:]
).order_by('-created')[settings.AUTH_TOKEN_PER_USER:]
return invalid_tokens
def generate_key(self):
@@ -313,7 +312,7 @@ class AuthToken(BaseModel):
valid_n_tokens_qs = self.user.auth_tokens.filter(
expires__gt=now,
reason='',
).order_by('-created')[0:tower_settings.AUTH_TOKEN_PER_USER]
).order_by('-created')[0:settings.AUTH_TOKEN_PER_USER]
valid_n_tokens = valid_n_tokens_qs.values_list('key', flat=True)
return bool(self.key in valid_n_tokens)

3
awx/main/models/projects.py
View File

@@ -28,7 +28,6 @@ from awx.main.models.unified_jobs import * # noqa
from awx.main.models.mixins import ResourceMixin
from awx.main.utils import update_scm_url
from awx.main.fields import ImplicitRoleField
from awx.main.conf import tower_settings
from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR,
@@ -433,7 +432,7 @@ class ProjectUpdate(UnifiedJob, ProjectOptions, JobNotificationMixin):
return reverse('api:project_update_detail', args=(self.pk,))
def get_ui_url(self):
return urlparse.urljoin(tower_settings.TOWER_URL_BASE, "/#/scm_update/{}".format(self.pk))
return urlparse.urljoin(settings.TOWER_URL_BASE, "/#/scm_update/{}".format(self.pk))
def _update_parent_instance(self):
parent_instance = self._get_parent_instance()

2
awx/main/models/rbac.py
View File

@@ -381,7 +381,7 @@ class Role(models.Model):
'ancestors_table': Role.ancestors.through._meta.db_table,
'parents_table': Role.parents.through._meta.db_table,
'roles_table': Role._meta.db_table,
'ids': ','.join(str(x) for x in user.roles.values_list('id', flat=True))
'ids': ','.join(str(x) for x in user.roles.values_list('id', flat=True)),
}
qs = Role.objects.extra(

34
awx/main/models/unified_jobs.py
View File

@@ -800,34 +800,43 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
status=self.status,
traceback=self.result_traceback)
def start(self, error_callback, success_callback, **kwargs):
'''
Start the task running via Celery.
'''
task_class = self._get_task_class()
def pre_start(self, **kwargs):
if not self.can_start:
self.job_explanation = u'%s is not in a startable status: %s, expecting one of %s' % (self._meta.verbose_name, self.status, str(('new', 'waiting')))
self.save(update_fields=['job_explanation'])
return False
return (False, None)
needed = self.get_passwords_needed_to_start()
try:
start_args = json.loads(decrypt_field(self, 'start_args'))
except Exception:
start_args = None
if start_args in (None, ''):
start_args = kwargs
opts = dict([(field, start_args.get(field, '')) for field in needed])
if not all(opts.values()):
missing_fields = ', '.join([k for k,v in opts.items() if not v])
self.job_explanation = u'Missing needed fields: %s.' % missing_fields
self.save(update_fields=['job_explanation'])
return False
#extra_data = dict([(field, kwargs[field]) for field in kwargs
# if field not in needed])
return (False, None)
if 'extra_vars' in kwargs:
self.handle_extra_data(kwargs['extra_vars'])
task_class().apply_async((self.pk,), opts, link_error=error_callback, link=success_callback)
return True
return (True, opts)
def start(self, error_callback, success_callback, **kwargs):
'''
Start the task running via Celery.
'''
task_class = self._get_task_class()
(res, opts) = self.pre_start(**kwargs)
if res:
task_class().apply_async((self.pk,), opts, link_error=error_callback, link=success_callback)
return res
def signal_start(self, **kwargs):
"""Notify the task runner system to begin work on this task."""
@@ -854,6 +863,9 @@ class UnifiedJob(PolymorphicModel, PasswordFieldsModel, CommonModelNameNotUnique
self.update_fields(start_args=json.dumps(kwargs), status='pending')
self.websocket_emit_status("pending")
from awx.main.scheduler.tasks import run_job_launch
run_job_launch.delay(self.id)
# Each type of unified job has a different Task class; get the
# appropirate one.
# task_type = get_type_for_model(self)

8
awx/main/models/workflow.py
View File

@@ -240,3 +240,11 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, JobNotificationMixin, Workflow
def get_notification_friendly_name(self):
return "Workflow Job"
def start(self, *args, **kwargs):
(res, opts) = self.pre_start(**kwargs)
if res:
self.status = 'running'
self.save()
self.socketio_emit_status("running")
return res

7
awx/main/registrar.py
View File

@@ -1,11 +1,8 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
import logging
from django.db.models.signals import pre_save, post_save, pre_delete, m2m_changed
logger = logging.getLogger('awx.main.registrar')
class ActivityStreamRegistrar(object):
@@ -13,9 +10,7 @@ class ActivityStreamRegistrar(object):
self.models = []
def connect(self, model):
from awx.main.conf import tower_settings
if not getattr(tower_settings, 'ACTIVITY_STREAM_ENABLED', True):
return
# Always register model; the signal handlers will check if activity stream is enabled.
from awx.main.signals import activity_stream_create, activity_stream_update, activity_stream_delete, activity_stream_associate
if model not in self.models:

257
awx/main/scheduler/__init__.py Normal file
View File

@@ -0,0 +1,257 @@
#Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
# Python
import datetime
import logging
# Django
from django.conf import settings
from django.db import transaction
# AWX
from awx.main.models import * # noqa
from awx.main.utils import get_system_task_capacity
from awx.main.scheduler.dag_simple import SimpleDAG
from awx.main.scheduler.dag_workflow import WorkflowDAG
# Celery
from celery.task.control import inspect
logger = logging.getLogger('awx.main.scheduler')
def get_tasks():
"""Fetch all Tower tasks that are relevant to the task management
system.
"""
RELEVANT_JOBS = ('pending', 'waiting', 'running')
# TODO: Replace this when we can grab all objects in a sane way.
graph_jobs = [j for j in Job.objects.filter(status__in=RELEVANT_JOBS)]
graph_ad_hoc_commands = [ahc for ahc in AdHocCommand.objects.filter(status__in=RELEVANT_JOBS)]
graph_inventory_updates = [iu for iu in
InventoryUpdate.objects.filter(status__in=RELEVANT_JOBS)]
graph_project_updates = [pu for pu in
ProjectUpdate.objects.filter(status__in=RELEVANT_JOBS)]
graph_system_jobs = [sj for sj in
SystemJob.objects.filter(status__in=RELEVANT_JOBS)]
graph_workflow_jobs = [wf for wf in
WorkflowJob.objects.filter(status__in=RELEVANT_JOBS)]
all_actions = sorted(graph_jobs + graph_ad_hoc_commands + graph_inventory_updates +
graph_project_updates + graph_system_jobs +
graph_workflow_jobs,
key=lambda task: task.created)
return all_actions
def get_running_workflow_jobs():
graph_workflow_jobs = [wf for wf in
WorkflowJob.objects.filter(status='running')]
return graph_workflow_jobs
def spawn_workflow_graph_jobs(workflow_jobs):
# TODO: Consider using transaction.atomic
for workflow_job in workflow_jobs:
dag = WorkflowDAG(workflow_job)
spawn_nodes = dag.bfs_nodes_to_run()
for spawn_node in spawn_nodes:
# TODO: Inject job template template params as kwargs.
# Make sure to take into account extra_vars merge logic
kv = {}
job = spawn_node.unified_job_template.create_unified_job(**kv)
spawn_node.job = job
spawn_node.save()
can_start = job.signal_start(**kv)
if not can_start:
job.status = 'failed'
job.job_explanation = "Workflow job could not start because it was not in the right state or required manual credentials"
job.save(update_fields=['status', 'job_explanation'])
job.socketio_emit_status("failed")
# TODO: should we emit a status on the socket here similar to tasks.py tower_periodic_scheduler() ?
#emit_websocket_notification('/socket.io/jobs', '', dict(id=))
# See comment in tasks.py::RunWorkflowJob::run()
def process_finished_workflow_jobs(workflow_jobs):
for workflow_job in workflow_jobs:
dag = WorkflowDAG(workflow_job)
if dag.is_workflow_done():
with transaction.atomic():
# TODO: detect if wfj failed
workflow_job.status = 'completed'
workflow_job.save()
workflow_job.socketio_emit_status('completed')
def rebuild_graph():
"""Regenerate the task graph by refreshing known tasks from Tower, purging
orphaned running tasks, and creating dependencies for new tasks before
generating directed edge relationships between those tasks.
"""
'''
# Sanity check: Only do this on the primary node.
if Instance.objects.my_role() == 'secondary':
return None
'''
inspector = inspect()
if not hasattr(settings, 'IGNORE_CELERY_INSPECTOR'):
active_task_queues = inspector.active()
else:
logger.warn("Ignoring celery task inspector")
active_task_queues = None
all_sorted_tasks = get_tasks()
if not len(all_sorted_tasks):
return None
active_tasks = []
if active_task_queues is not None:
for queue in active_task_queues:
active_tasks += [at['id'] for at in active_task_queues[queue]]
else:
logger.error("Could not communicate with celery!")
# TODO: Something needs to be done here to signal to the system
# as a whole that celery appears to be down.
if not hasattr(settings, 'CELERY_UNIT_TEST'):
return None
running_tasks = filter(lambda t: t.status == 'running', all_sorted_tasks)
running_celery_tasks = filter(lambda t: type(t) != WorkflowJob, running_tasks)
waiting_tasks = filter(lambda t: t.status != 'running', all_sorted_tasks)
new_tasks = filter(lambda t: t.status == 'pending', all_sorted_tasks)
# Check running tasks and make sure they are active in celery
logger.debug("Active celery tasks: " + str(active_tasks))
for task in list(running_celery_tasks):
if (task.celery_task_id not in active_tasks and not hasattr(settings, 'IGNORE_CELERY_INSPECTOR')):
# NOTE: Pull status again and make sure it didn't finish in
# the meantime?
task.status = 'failed'
task.job_explanation += ' '.join((
'Task was marked as running in Tower but was not present in',
'Celery, so it has been marked as failed.',
))
task.save()
task.socketio_emit_status("failed")
running_tasks.pop(task)
logger.error("Task %s appears orphaned... marking as failed" % task)
# Create and process dependencies for new tasks
for task in new_tasks:
logger.debug("Checking dependencies for: %s" % str(task))
try:
task_dependencies = task.generate_dependencies(running_tasks + waiting_tasks)
except Exception, e:
logger.error("Failed processing dependencies for {}: {}".format(task, e))
task.status = 'failed'
task.job_explanation += 'Task failed to generate dependencies: {}'.format(e)
task.save()
task.socketio_emit_status("failed")
continue
logger.debug("New dependencies: %s" % str(task_dependencies))
for dep in task_dependencies:
# We recalculate the created time for the moment to ensure the
# dependencies are always sorted in the right order relative to
# the dependent task.
time_delt = len(task_dependencies) - task_dependencies.index(dep)
dep.created = task.created - datetime.timedelta(seconds=1 + time_delt)
dep.status = 'waiting'
dep.save()
waiting_tasks.insert(waiting_tasks.index(task), dep)
if not hasattr(settings, 'UNIT_TEST_IGNORE_TASK_WAIT'):
task.status = 'waiting'
task.save()
# Rebuild graph
graph = SimpleDAG()
for task in running_tasks:
graph.add_node(task)
for wait_task in waiting_tasks[:50]:
node_dependencies = []
for node in graph:
if wait_task.is_blocked_by(node['node_object']):
node_dependencies.append(node['node_object'])
graph.add_node(wait_task)
for dependency in node_dependencies:
graph.add_edge(wait_task, dependency)
if settings.DEBUG:
graph.generate_graphviz_plot()
return graph
def process_graph(graph, task_capacity):
"""Given a task dependency graph, start and manage tasks given their
priority and weight.
"""
from awx.main.tasks import handle_work_error, handle_work_success
leaf_nodes = graph.get_leaf_nodes()
running_nodes = filter(lambda x: x['node_object'].status == 'running', leaf_nodes)
running_impact = sum([t['node_object'].task_impact for t in running_nodes])
ready_nodes = filter(lambda x: x['node_object'].status != 'running', leaf_nodes)
remaining_volume = task_capacity - running_impact
logger.info('Running Nodes: %s; Capacity: %s; Running Impact: %s; '
'Remaining Capacity: %s' %
(str(running_nodes), str(task_capacity),
str(running_impact), str(remaining_volume)))
logger.info("Ready Nodes: %s" % str(ready_nodes))
for task_node in ready_nodes:
node_obj = task_node['node_object']
# NOTE: This could be used to pass metadata through the task system
# node_args = task_node['metadata']
impact = node_obj.task_impact
if impact <= remaining_volume or running_impact == 0:
node_dependencies = graph.get_dependents(node_obj)
# Allow other tasks to continue if a job fails, even if they are
# other jobs.
node_type = graph.get_node_type(node_obj)
if node_type == 'job':
# clear dependencies because a job can block (not necessarily
# depend) on other jobs that share the same job template
node_dependencies = []
# Make the workflow_job look like it's started by setting status to
# running, but don't make a celery Task for it.
# Introduce jobs from the workflow so they are candidates to run.
# Call process_graph() again to allow choosing for run, the
# created candidate jobs.
elif node_type == 'workflow_job':
node_obj.start()
spawn_workflow_graph_jobs([node_obj])
return process_graph(graph, task_capacity)
dependent_nodes = [{'type': graph.get_node_type(node_obj), 'id': node_obj.id}] + \
[{'type': graph.get_node_type(n['node_object']),
'id': n['node_object'].id} for n in node_dependencies]
error_handler = handle_work_error.s(subtasks=dependent_nodes)
success_handler = handle_work_success.s(task_actual={'type': graph.get_node_type(node_obj),
'id': node_obj.id})
with transaction.atomic():
start_status = node_obj.start(error_callback=error_handler, success_callback=success_handler)
if not start_status:
node_obj.status = 'failed'
if node_obj.job_explanation:
node_obj.job_explanation += ' '
node_obj.job_explanation += 'Task failed pre-start check.'
node_obj.save()
continue
remaining_volume -= impact
running_impact += impact
logger.info('Started Node: %s (capacity hit: %s) '
'Remaining Capacity: %s' %
(str(node_obj), str(impact), str(remaining_volume)))
def schedule():
with transaction.atomic():
# Lock
Instance.objects.select_for_update().all()[0]
task_capacity = get_system_task_capacity()
workflow_jobs = get_running_workflow_jobs()
process_finished_workflow_jobs(workflow_jobs)
spawn_workflow_graph_jobs(workflow_jobs)
graph = rebuild_graph()
if graph:
process_graph(graph, task_capacity)
# Unlock, due to transaction ending

140
awx/main/scheduler/dag_simple.py Normal file
View File

@@ -0,0 +1,140 @@
from awx.main.models import (
Job,
AdHocCommand,
InventoryUpdate,
ProjectUpdate,
WorkflowJob,
SystemJob,
)
class SimpleDAG(object):
''' A simple implementation of a directed acyclic graph '''
def __init__(self):
self.nodes = []
self.edges = []
def __contains__(self, obj):
for node in self.nodes:
if node['node_object'] == obj:
return True
return False
def __len__(self):
return len(self.nodes)
def __iter__(self):
return self.nodes.__iter__()
def generate_graphviz_plot(self):
def short_string_obj(obj):
if type(obj) == Job:
type_str = "Job"
if type(obj) == AdHocCommand:
type_str = "AdHocCommand"
elif type(obj) == InventoryUpdate:
type_str = "Inventory"
elif type(obj) == ProjectUpdate:
type_str = "Project"
elif type(obj) == WorkflowJob:
type_str = "Workflow"
else:
type_str = "Unknown"
type_str += "%s" % str(obj.id)
return type_str
doc = """
digraph g {
rankdir = LR
"""
for n in self.nodes:
doc += "%s [color = %s]\n" % (
short_string_obj(n['node_object']),
"red" if n['node_object'].status == 'running' else "black",
)
for from_node, to_node, label in self.edges:
doc += "%s -> %s [ label=\"%s\" ];\n" % (
short_string_obj(self.nodes[from_node]['node_object']),
short_string_obj(self.nodes[to_node]['node_object']),
label,
)
doc += "}\n"
gv_file = open('/tmp/graph.gv', 'w')
gv_file.write(doc)
gv_file.close()
def add_node(self, obj, metadata=None):
if self.find_ord(obj) is None:
self.nodes.append(dict(node_object=obj, metadata=metadata))
def add_edge(self, from_obj, to_obj, label=None):
from_obj_ord = self.find_ord(from_obj)
to_obj_ord = self.find_ord(to_obj)
if from_obj_ord is None or to_obj_ord is None:
raise LookupError("Object not found")
self.edges.append((from_obj_ord, to_obj_ord, label))
def add_edges(self, edgelist):
for edge_pair in edgelist:
self.add_edge(edge_pair[0], edge_pair[1], edge_pair[2])
def find_ord(self, obj):
for idx in range(len(self.nodes)):
if obj == self.nodes[idx]['node_object']:
return idx
return None
def get_node_type(self, obj):
if type(obj) == Job:
return "job"
elif type(obj) == AdHocCommand:
return "ad_hoc_command"
elif type(obj) == InventoryUpdate:
return "inventory_update"
elif type(obj) == ProjectUpdate:
return "project_update"
elif type(obj) == SystemJob:
return "system_job"
elif type(obj) == WorkflowJob:
return "workflow_job"
return "unknown"
def get_dependencies(self, obj, label=None):
antecedents = []
this_ord = self.find_ord(obj)
for node, dep, lbl in self.edges:
if label:
if node == this_ord and lbl == label:
antecedents.append(self.nodes[dep])
else:
if node == this_ord:
antecedents.append(self.nodes[dep])
return antecedents
def get_dependents(self, obj, label=None):
decendents = []
this_ord = self.find_ord(obj)
for node, dep, lbl in self.edges:
if label:
if dep == this_ord and lbl == label:
decendents.append(self.nodes[node])
else:
if dep == this_ord:
decendents.append(self.nodes[node])
return decendents
def get_leaf_nodes(self):
leafs = []
for n in self.nodes:
if len(self.get_dependencies(n['node_object'])) < 1:
leafs.append(n)
return leafs
def get_root_nodes(self):
roots = []
for n in self.nodes:
if len(self.get_dependents(n['node_object'])) < 1:
roots.append(n)
return roots

72
awx/main/scheduler/dag_workflow.py Normal file
View File

@@ -0,0 +1,72 @@
# AWX
from awx.main.scheduler.dag_simple import SimpleDAG
class WorkflowDAG(SimpleDAG):
def __init__(self, workflow_job=None):
super(WorkflowDAG, self).__init__()
if workflow_job:
self._init_graph(workflow_job)
def _init_graph(self, workflow_job):
workflow_nodes = workflow_job.workflow_job_nodes.all()
for workflow_node in workflow_nodes:
self.add_node(workflow_node)
for node_type in ['success_nodes', 'failure_nodes', 'always_nodes']:
for workflow_node in workflow_nodes:
related_nodes = getattr(workflow_node, node_type).all()
for related_node in related_nodes:
self.add_edge(workflow_node, related_node, node_type)
def bfs_nodes_to_run(self):
root_nodes = self.get_root_nodes()
nodes = root_nodes
nodes_found = []
for index, n in enumerate(nodes):
obj = n['node_object']
job = obj.job
if not job:
nodes_found.append(n)
# Job is about to run or is running. Hold our horses and wait for
# the job to finish. We can't proceed down the graph path until we
# have the job result.
elif job.status not in ['failed', 'error', 'successful']:
continue
elif job.status in ['failed', 'error']:
children_failed = self.get_dependencies(obj, 'failure_nodes')
children_always = self.get_dependencies(obj, 'always_nodes')
children_all = children_failed + children_always
nodes.extend(children_all)
elif job.status in ['successful']:
children_success = self.get_dependencies(obj, 'success_nodes')
nodes.extend(children_success)
return [n['node_object'] for n in nodes_found]
def is_workflow_done(self):
root_nodes = self.get_root_nodes()
nodes = root_nodes
for index, n in enumerate(nodes):
obj = n['node_object']
job = obj.job
if not job:
return False
# Job is about to run or is running. Hold our horses and wait for
# the job to finish. We can't proceed down the graph path until we
# have the job result.
elif job.status not in ['failed', 'error', 'successful']:
return False
elif job.status in ['failed', 'error']:
children_failed = self.get_dependencies(obj, 'failure_nodes')
children_always = self.get_dependencies(obj, 'always_nodes')
children_all = children_failed + children_always
nodes.extend(children_all)
elif job.status in ['successful']:
children_success = self.get_dependencies(obj, 'success_nodes')
nodes.extend(children_success)
return True

79
awx/main/scheduler/tasks.py Normal file
View File

@@ -0,0 +1,79 @@
# Python
import logging
import time
# Celery
from celery import task
# AWX
from awx.main.models import UnifiedJob
from awx.main.scheduler import schedule
logger = logging.getLogger('awx.main.scheduler')
# TODO: move logic to UnifiedJob model and use bind=True feature of celery.
# Would we need the request loop then? I think so. Even if we get the in-memory
# updated model, the call to schedule() may get stale data.
@task
def run_job_launch(job_id):
# Wait for job to exist.
# The job is created in a transaction then the message is created, but
# the transaction may not have completed.
# FIXME: We could generate the message in a Django signal handler.
# OR, we could call an explicit commit in the view and then send the
# message.
retries = 10
retry = 0
while not UnifiedJob.objects.filter(id=job_id).exists():
time.sleep(0.3)
if retry >= retries:
logger.error("Failed to process 'job_launch' message for job %d" % job_id)
# ack the message so we don't build up the queue.
#
# The job can still be chosen to run during tower startup or
# when another job is started or completes
return
retry += 1
# "Safe" to get the job now since it exists.
# Really, there is a race condition from exists to get
# TODO: while not loop should call get wrapped in a try except
#job = UnifiedJob.objects.get(id=job_id)
schedule()
@task
def run_job_complete(job_id):
# TODO: use list of finished status from jobs.py or unified_jobs.py
finished_status = ['successful', 'error', 'failed', 'completed']
q = UnifiedJob.objects.filter(id=job_id)
# Ensure that the job is updated in the database before we call to
# schedule the next job.
retries = 10
retry = 0
while True:
# Job not found, most likely deleted. That's fine
if not q.exists():
logger.warn("Failed to find job '%d' while processing 'job_complete' message. Presume that it was deleted." % job_id)
break
job = q[0]
if job.status in finished_status:
break
time.sleep(0.3)
if retry >= retries:
logger.error("Expected job status '%s' to be one of '%s' while processing 'job_complete' message." % (job.status, finished_status))
return
retry += 1
schedule()

11
awx/main/signals.py
View File

@@ -8,6 +8,7 @@ import threading
import json
# Django
from django.conf import settings
from django.db.models.signals import post_save, pre_delete, post_delete, m2m_changed
from django.dispatch import receiver
@@ -21,7 +22,6 @@ from awx.api.serializers import * # noqa
from awx.main.utils import model_instance_diff, model_to_dict, camelcase_to_underscore
from awx.main.utils import ignore_inventory_computed_fields, ignore_inventory_group_removal, _inventory_updates
from awx.main.tasks import update_inventory_computed_fields
from awx.main.conf import tower_settings
from awx.main.consumers import emit_channel_notification
@@ -302,10 +302,10 @@ def update_host_last_job_after_job_deleted(sender, **kwargs):
class ActivityStreamEnabled(threading.local):
def __init__(self):
self.enabled = getattr(tower_settings, 'ACTIVITY_STREAM_ENABLED', True)
self.enabled = True
def __nonzero__(self):
return bool(self.enabled)
return bool(self.enabled and getattr(settings, 'ACTIVITY_STREAM_ENABLED', True))
activity_stream_enabled = ActivityStreamEnabled()
@@ -335,7 +335,6 @@ model_serializer_mapping = {
JobTemplate: JobTemplateSerializer,
Job: JobSerializer,
AdHocCommand: AdHocCommandSerializer,
TowerSettings: TowerSettingsSerializer,
NotificationTemplate: NotificationTemplateSerializer,
Notification: NotificationSerializer,
}
@@ -359,7 +358,7 @@ def activity_stream_create(sender, instance, created, **kwargs):
#TODO: Weird situation where cascade SETNULL doesn't work
# it might actually be a good idea to remove all of these FK references since
# we don't really use them anyway.
if type(instance) is not TowerSettings:
if instance._meta.model_name != 'setting': # Is not conf.Setting instance
getattr(activity_entry, object1).add(instance)
def activity_stream_update(sender, instance, **kwargs):
@@ -382,7 +381,7 @@ def activity_stream_update(sender, instance, **kwargs):
object1=object1,
changes=json.dumps(changes))
activity_entry.save()
if type(instance) is not TowerSettings:
if instance._meta.model_name != 'setting': # Is not conf.Setting instance
getattr(activity_entry, object1).add(instance)
def activity_stream_delete(sender, instance, **kwargs):

2
awx/main/south_migrations/0071_v240_changes.py
View File

@@ -4,7 +4,7 @@ from south.db import db
from south.v2 import DataMigration
from django.db import models
from django.utils.timezone import now
from awx.api.license import feature_enabled
from awx.conf.license import feature_enabled
class Migration(DataMigration):

83
awx/main/tasks.py
View File

@@ -47,9 +47,7 @@ from django.contrib.auth.models import User
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.models import * # noqa
from awx.main.models import UnifiedJob
from awx.main.queue import FifoQueue
from awx.main.conf import tower_settings
from awx.main.task_engine import TaskSerializer, TASK_TIMEOUT_INTERVAL
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import (get_ansible_version, get_ssh_version, decrypt_field, update_scm_url,
check_proot_installed, build_proot_temp_dir, wrap_args_with_proot)
from awx.main.consumers import emit_channel_notification
@@ -58,7 +56,7 @@ __all__ = ['RunJob', 'RunSystemJob', 'RunProjectUpdate', 'RunInventoryUpdate',
'RunAdHocCommand', 'RunWorkflowJob', 'handle_work_error',
'handle_work_success', 'update_inventory_computed_fields',
'send_notifications', 'run_administrative_checks',
'run_workflow_job']
'RunJobLaunch']
HIDDEN_PASSWORD = '**********'
@@ -105,10 +103,9 @@ def send_notifications(notification_list, job_id=None):
@task(bind=True, queue='default')
def run_administrative_checks(self):
if not tower_settings.TOWER_ADMIN_ALERTS:
if not settings.TOWER_ADMIN_ALERTS:
return
reader = TaskSerializer()
validation_info = reader.from_database()
validation_info = TaskEnhancer().validate_enhancements()
if validation_info.get('instance_count', 0) < 1:
return
used_percentage = float(validation_info.get('current_instances', 0)) / float(validation_info.get('instance_count', 100))
@@ -118,7 +115,7 @@ def run_administrative_checks(self):
"Ansible Tower host usage over 90%",
tower_admin_emails,
fail_silently=True)
if validation_info.get('time_remaining', 0) < TASK_TIMEOUT_INTERVAL:
if validation_info.get('date_warning', False):
send_mail("Ansible Tower license will expire soon",
"Ansible Tower license will expire soon",
tower_admin_emails,
@@ -179,14 +176,6 @@ def tower_periodic_scheduler(self):
new_unified_job.websocket_emit_status("failed")
emit_channel_notification('schedules-changed', dict(id=schedule.id, group_name="schedules"))
@task(queue='default')
def notify_task_runner(metadata_dict):
"""Add the given task into the Tower task manager's queue, to be consumed
by the task system.
"""
queue = FifoQueue('tower_task_manager')
queue.push(metadata_dict)
def _send_notification_templates(instance, status_str):
if status_str not in ['succeeded', 'failed']:
raise ValueError("status_str must be either succeeded or failed")
@@ -202,6 +191,7 @@ def _send_notification_templates(instance, status_str):
for n in all_notification_templates],
job_id=instance.id)
@task(bind=True, queue='default')
def handle_work_success(self, result, task_actual):
instance = UnifiedJob.get_instance_by_type(task_actual['type'], task_actual['id'])
@@ -210,6 +200,9 @@ def handle_work_success(self, result, task_actual):
_send_notification_templates(instance, 'succeeded')
from awx.main.scheduler.tasks import run_job_complete
run_job_complete.delay(instance.id)
@task(bind=True, queue='default')
def handle_work_error(self, task_id, subtasks=None):
print('Executing error task id %s, subtasks: %s' %
@@ -235,15 +228,18 @@ def handle_work_error(self, task_id, subtasks=None):
(first_instance_type, first_instance.name, first_instance.id)
instance.save()
instance.websocket_emit_status("failed")
notification_body = first_task.notification_data()
notification_subject = "{} #{} '{}' failed on Ansible Tower: {}".format(first_task_friendly_name,
first_task_id,
smart_str(first_task_name),
notification_body['url'])
notification_body['friendly_name'] = first_task_friendly_name
send_notifications.delay([n.generate_notification(notification_subject, notification_body).id
for n in set(notification_templates.get('error', []) + notification_templates.get('any', []))],
job_id=first_task_id)
if first_instance:
_send_notification_templates(first_instance, 'failed')
# We only send 1 job complete message since all the job completion message
# handling does is trigger the scheduler. If we extend the functionality of
# what the job complete message handler does then we may want to send a
# completion event for each job here.
if first_instance:
from awx.main.scheduler.tasks import run_job_complete
run_job_complete.delay(first_instance.id)
pass
@task(queue='default')
def update_inventory_computed_fields(inventory_id, should_update_hosts=True):
@@ -423,7 +419,7 @@ class BaseTask(Task):
# NOTE:
# Derived class should call add_ansible_venv() or add_tower_venv()
if self.should_use_proot(instance, **kwargs):
env['PROOT_TMP_DIR'] = tower_settings.AWX_PROOT_BASE_PATH
env['PROOT_TMP_DIR'] = settings.AWX_PROOT_BASE_PATH
return env
def build_safe_env(self, instance, **kwargs):
@@ -536,7 +532,7 @@ class BaseTask(Task):
instance = self.update_model(instance.pk)
if instance.cancel_flag:
try:
if tower_settings.AWX_PROOT_ENABLED and self.should_use_proot(instance):
if settings.AWX_PROOT_ENABLED and self.should_use_proot(instance):
# NOTE: Refactor this once we get a newer psutil across the board
if not psutil:
os.kill(child.pid, signal.SIGKILL)
@@ -733,9 +729,9 @@ class RunJob(BaseTask):
'''
plugin_dir = self.get_path_to('..', 'plugins', 'callback')
plugin_dirs = [plugin_dir]
if hasattr(tower_settings, 'AWX_ANSIBLE_CALLBACK_PLUGINS') and \
tower_settings.AWX_ANSIBLE_CALLBACK_PLUGINS:
plugin_dirs.append(tower_settings.AWX_ANSIBLE_CALLBACK_PLUGINS)
if hasattr(settings, 'AWX_ANSIBLE_CALLBACK_PLUGINS') and \
settings.AWX_ANSIBLE_CALLBACK_PLUGINS:
plugin_dirs.extend(settings.AWX_ANSIBLE_CALLBACK_PLUGINS)
plugin_path = ':'.join(plugin_dirs)
env = super(RunJob, self).build_env(job, **kwargs)
env = self.add_ansible_venv(env)
@@ -950,7 +946,7 @@ class RunJob(BaseTask):
'''
Return whether this task should use proot.
'''
return getattr(tower_settings, 'AWX_PROOT_ENABLED', False)
return getattr(settings, 'AWX_PROOT_ENABLED', False)
def post_run_hook(self, job, **kwargs):
'''
@@ -1630,7 +1626,7 @@ class RunAdHocCommand(BaseTask):
'''
Return whether this task should use proot.
'''
return getattr(tower_settings, 'AWX_PROOT_ENABLED', False)
return getattr(settings, 'AWX_PROOT_ENABLED', False)
def post_run_hook(self, ad_hoc_command, **kwargs):
'''
@@ -1670,21 +1666,30 @@ class RunSystemJob(BaseTask):
def build_cwd(self, instance, **kwargs):
return settings.BASE_DIR
'''
class RunWorkflowJob(BaseTask):
name = 'awx.main.tasks.run_workflow_job'
model = WorkflowJob
def run(self, pk, **kwargs):
from awx.main.management.commands.run_task_system import WorkflowDAG
'''
Run the job/task and capture its output.
'''
pass
#Run the job/task and capture its output.
instance = self.update_model(pk, status='running', celery_task_id=self.request.id)
instance.socketio_emit_status("running")
# FIXME: Detect workflow run completion
# FIXME: Currently, the workflow job busy waits until the graph run is
# complete. Instead, the workflow job should return or never even run,
# because all of the "launch logic" can be done schedule().
# However, other aspects of our system depend on a 1-1 relationship
# between a Job and a Celery Task.
#
# * If we let the workflow job task (RunWorkflowJob.run()) complete
# then how do we trigger the handle_work_error and
# handle_work_success subtasks?
#
# * How do we handle the recovery process? (i.e. there is an entry in
# the database but not in celery).
while True:
dag = WorkflowDAG(instance)
if dag.is_workflow_done():
@@ -1694,4 +1699,4 @@ class RunWorkflowJob(BaseTask):
time.sleep(1)
instance.socketio_emit_status(instance.status)
# TODO: Handle cancel
'''

38
awx/main/tests/base.py
View File

@@ -12,7 +12,6 @@ import sys
import tempfile
import time
import urllib
from multiprocessing import Process
import re
import mock
@@ -30,9 +29,8 @@ from django.utils.encoding import force_text
# AWX
from awx.main.models import * # noqa
from awx.main.management.commands.run_task_system import run_taskmanager
from awx.main.task_engine import TaskEnhancer
from awx.main.utils import get_ansible_version
from awx.main.task_engine import TaskEngager as LicenseWriter
from awx.sso.backends import LDAPSettings
from awx.main.tests.URI import URI # noqa
@@ -143,35 +141,25 @@ class BaseTestMixin(MockCommonlySlowTestMixin):
return __name__ + '-generated-' + string + rnd_str
def create_test_license_file(self, instance_count=10000, license_date=int(time.time() + 3600), features=None):
writer = LicenseWriter(
settings.LICENSE = TaskEnhancer(
company_name='AWX',
contact_name='AWX Admin',
contact_email='awx@example.com',
license_date=license_date,
instance_count=instance_count,
license_type='enterprise',
features=features)
handle, license_path = tempfile.mkstemp(suffix='.json')
os.close(handle)
writer.write_file(license_path)
self._temp_paths.append(license_path)
os.environ['AWX_LICENSE_FILE'] = license_path
cache.clear()
features=features,
).enhance()
def create_basic_license_file(self, instance_count=100, license_date=int(time.time() + 3600)):
writer = LicenseWriter(
settings.LICENSE = TaskEnhancer(
company_name='AWX',
contact_name='AWX Admin',
contact_email='awx@example.com',
license_date=license_date,
instance_count=instance_count,
license_type='basic')
handle, license_path = tempfile.mkstemp(suffix='.json')
os.close(handle)
writer.write_file(license_path)
self._temp_paths.append(license_path)
os.environ['AWX_LICENSE_FILE'] = license_path
cache.clear()
license_type='basic',
).enhance()
def create_expired_license_file(self, instance_count=1000, grace_period=False):
license_date = time.time() - 1
@@ -654,18 +642,6 @@ class BaseTestMixin(MockCommonlySlowTestMixin):
u'expected no traceback, got:\n%s' %
job.result_traceback)
def start_taskmanager(self, command_port):
self.start_redis()
self.taskmanager_process = Process(target=run_taskmanager,
args=(command_port,))
self.taskmanager_process.start()
def terminate_taskmanager(self):
if hasattr(self, 'taskmanager_process'):
self.taskmanager_process.terminate()
self.stop_redis()
class BaseTest(BaseTestMixin, django.test.TestCase):
'''
Base class for unit tests.

28
awx/main/tests/data/ssh.py
View File

@@ -1,3 +1,31 @@
TEST_SSH_RSA1_KEY_DATA = '''-----BEGIN PRIVATE KEY-----
uFZFyag7VVqI+q/oGnQu+wj/pMi5ox+Qz5L3W0D745DzwgDXOeObAfNlr9NtIKbn
sZ5E0+rYB4Q/U0CYr5juNJQV1dbxq2Em1160axboe2QbvX6wE6Sm6wW9b9cr+PoF
MoYQebUnCY0ObrLbrRugSfZc17lyxK0ZGRgPXKhpMg6Ecv8XpvhjUYU9Esyqfuco
/p26Q140/HsHeHYNma0dQHCEjMr/qEzOY1qguHj+hRf3SARtM9Q+YNgpxchcDDVS
O+n+8Ljd/p82bpEJwxmpXealeWbI6gB9/R6wcCL+ZyCZpnHJd/NJ809Vtu47ZdDi
E6jvqS/3AQhuQKhJlLSDIzezB2VKKrHwOvHkg/+uLoCqHN34Gk6Qio7x69SvXy88
a7q9D1l/Zx60o08FyZyqlo7l0l/r8EY+36cuI/lvAvfxc5VHVEOvKseUjFRBiCv9
MkKNxaScoYsPwY7SIS6gD93tg3eM5pA0nfMfya9u1+uq/QCM1gNG3mm6Zd8YG4c/
Dx4bmsj8cp5ni/Ffl/sKzKYq1THunJEFGXOZRibdxk/Fal3SQrRAwy7CgLQL8SMh
IWqcFm25OtSOP1r1LE25t5pQsMdmp0IP2fEF0t/pXPm1ZfrTurPMqpo4FGm2hkki
U3sH/o6nrkSOjklOLWlwtTkkL4dWPlNwc8OYj8zFizXJkAfv1spzhv3lRouNkw4N
Mm22W7us2f3Ob0H5C07k26h6VuXX+0AybD4tIIcUXCLoNTqA0HvqhKpEuHu3Ck10
RaB8xHTxgwdhGVaNHMfy9B9l4tNs3Tb5k0LyeRRGVDhWCFo6axYULYebkj+hFLLY
+JE5RzPDFpTf1xbuT+e56H/lLFCUdDu0bn+D0W4ifXaVFegak4r6O4B53CbMqr+R
t6qDPKLUIuVJXK0J6Ay6XgmheXJGbgKh4OtDsc06gsTCE1nY4f/Z82AQahPBfTtF
J2z+NHdsLPn//HlxspGQtmLpuS7Wx0HYXZ+kPRSiE/vmITw85R2u8JSHQicVNN4C
2rlUo15TIU3tTx+WUIrHKHPidUNNotRb2p9n9FoSidU6upKnQHAT/JNv/zcvaia3
Bhl/wagheWTDnFKSmJ4HlKxplM/32h6MfHqsMVOl4F6eZWKaKgSgN8doXyFJo+sc
yAC6S0gJlD2gQI24iTI4Du1+UGh2MGb69eChvi5mbbdesaZrlR1dRqZpHG+6ob4H
nYLndRvobXS5l6pgGTDRYoUgSbQe21a7Uf3soGl5jHqLWc1zEPwrxV7Wr31mApr6
8VtGZcLSr0691Q1NLO3eIfuhbMN2mssX/Sl4t+4BibaucNIMfmhKQi8uHtwAXb47
+TMFlG2EQhZULFM4fLdF1vaizInU3cBk8lsz8i71tDc+5VQTEwoEB7Gksy/XZWEt
6SGHxXUDtNYa+G2O+sQhgqBjLIkVTV6KJOpvNZM+s8Vzv8qoFnD7isKBBrRvF1bP
GOXEG1jd7nSR0WSwcMCHGOrFEELDQPw3k5jqEdPFgVODoZPr+drZVnVz5SAGBk5Y
wsCNaDW+1dABYFlqRTepP5rrSu9wHnRAZ3ZGv+DHoGqenIC5IBR0sQ==
-----END PRIVATE KEY-----'''
TEST_SSH_KEY_DATA = '''-----BEGIN RSA PRIVATE KEY-----
MIIEpQIBAAKCAQEAyQ8F5bbgjHvk4SZJsKI9OmJKMFxZqRhvx4LaqjLTKbBwRBsY
1/C00NPiZn70dKbeyV7RNVZxuzM6yd3D3lwTdbDu/eJ0x72t3ch+TdLt/aenyy10

33
awx/main/tests/functional/api/test_activity_streams.py
View File

@@ -6,28 +6,27 @@ from awx.main.models.activity_stream import ActivityStream
from awx.main.access import ActivityStreamAccess
from django.core.urlresolvers import reverse
from django.conf import settings
def mock_feature_enabled(feature, bypass_database=None):
def mock_feature_enabled(feature):
return True
@pytest.fixture
def activity_stream_entry(organization, org_admin):
return ActivityStream.objects.filter(organization__pk=organization.pk, user=org_admin, operation='associate').first()
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_get_activity_stream_list(monkeypatch, organization, get, user):
def test_get_activity_stream_list(monkeypatch, organization, get, user, settings):
settings.ACTIVITY_STREAM_ENABLED = True
url = reverse('api:activity_stream_list')
response = get(url, user('admin', True))
assert response.status_code == 200
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_basic_fields(monkeypatch, organization, get, user):
def test_basic_fields(monkeypatch, organization, get, user, settings):
settings.ACTIVITY_STREAM_ENABLED = True
u = user('admin', True)
activity_stream = ActivityStream.objects.filter(organization=organization).latest('pk')
activity_stream.actor = u
@@ -44,10 +43,10 @@ def test_basic_fields(monkeypatch, organization, get, user):
assert 'organization' in response.data['summary_fields']
assert response.data['summary_fields']['organization'][0]['name'] == 'test-org'
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_middleware_actor_added(monkeypatch, post, get, user):
def test_middleware_actor_added(monkeypatch, post, get, user, settings):
settings.ACTIVITY_STREAM_ENABLED = True
u = user('admin-poster', True)
url = reverse('api:organization_list')
@@ -66,21 +65,19 @@ def test_middleware_actor_added(monkeypatch, post, get, user):
assert response.status_code == 200
assert response.data['summary_fields']['actor']['username'] == 'admin-poster'
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_rbac_stream_resource_roles(activity_stream_entry, organization, org_admin):
def test_rbac_stream_resource_roles(activity_stream_entry, organization, org_admin, settings):
settings.ACTIVITY_STREAM_ENABLED = True
assert activity_stream_entry.user.first() == org_admin
assert activity_stream_entry.organization.first() == organization
assert activity_stream_entry.role.first() == organization.admin_role
assert activity_stream_entry.object_relationship_type == 'awx.main.models.organization.Organization.admin_role'
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_rbac_stream_user_roles(activity_stream_entry, organization, org_admin):
def test_rbac_stream_user_roles(activity_stream_entry, organization, org_admin, settings):
settings.ACTIVITY_STREAM_ENABLED = True
assert activity_stream_entry.user.first() == org_admin
assert activity_stream_entry.organization.first() == organization
assert activity_stream_entry.role.first() == organization.admin_role
@@ -88,9 +85,9 @@ def test_rbac_stream_user_roles(activity_stream_entry, organization, org_admin):
@pytest.mark.django_db
@pytest.mark.activity_stream_access
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
def test_stream_access_cant_change(activity_stream_entry, organization, org_admin):
def test_stream_access_cant_change(activity_stream_entry, organization, org_admin, settings):
settings.ACTIVITY_STREAM_ENABLED = True
access = ActivityStreamAccess(org_admin)
# These should always return false because the activity stream can not be edited
assert not access.can_add(activity_stream_entry)
@@ -99,12 +96,12 @@ def test_stream_access_cant_change(activity_stream_entry, organization, org_admi
@pytest.mark.django_db
@pytest.mark.activity_stream_access
@pytest.mark.skipif(not getattr(settings, 'ACTIVITY_STREAM_ENABLED', True), reason="Activity stream not enabled")
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
def test_stream_queryset_hides_shows_items(
activity_stream_entry, organization, user, org_admin,
project, org_credential, inventory, label, deploy_jobtemplate,
notification_template, group, host, team):
notification_template, group, host, team, settings):
settings.ACTIVITY_STREAM_ENABLED = True
# this user is not in any organizations and should not see any resource activity
no_access_user = user('no-access-user', False)
queryset = ActivityStreamAccess(no_access_user).get_queryset()

4
awx/main/tests/functional/api/test_fact_versions.py
View File

@@ -13,10 +13,10 @@ from awx.main.utils import timestamp_apiformat
from django.core.urlresolvers import reverse
from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
def mock_feature_enabled(feature):
return True
def mock_feature_disabled(feature, bypass_database=None):
def mock_feature_disabled(feature):
return False
def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), get_params={}, host_count=1):

4
awx/main/tests/functional/api/test_fact_view.py
View File

@@ -6,10 +6,10 @@ from awx.main.utils import timestamp_apiformat
from django.core.urlresolvers import reverse
from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
def mock_feature_enabled(feature):
return True
def mock_feature_disabled(feature, bypass_database=None):
def mock_feature_disabled(feature):
return False
# TODO: Consider making the fact_scan() fixture a Class, instead of a function, and move this method into it

4
awx/main/tests/functional/api/test_organizations.py
View File

@@ -99,7 +99,7 @@ def test_organization_inventory_list(organization, inventory_factory, get, alice
@pytest.mark.django_db
@mock.patch('awx.api.views.feature_enabled', lambda feature,bypass_db=None: True)
@mock.patch('awx.api.views.feature_enabled', lambda feature: True)
def test_create_organization(post, admin, alice):
new_org = {
'name': 'new org',
@@ -111,7 +111,7 @@ def test_create_organization(post, admin, alice):
@pytest.mark.django_db
@mock.patch('awx.api.views.feature_enabled', lambda feature,bypass_db=None: True)
@mock.patch('awx.api.views.feature_enabled', lambda feature: True)
def test_create_organization_xfail(post, alice):
new_org = {
'name': 'new org',

6
awx/main/tests/functional/api/test_rbac_displays.py
View File

@@ -221,6 +221,12 @@ class TestAccessListCapabilities:
direct_access_list = response.data['results'][0]['summary_fields']['direct_access']
assert direct_access_list[0]['role']['user_capabilities']['unattach'] == 'foobar'
def test_user_access_list_direct_access_capability(self, rando, get):
"When a user views their own access list, they can not unattach their admin role"
response = get(reverse('api:user_access_list', args=(rando.id,)), rando)
direct_access_list = response.data['results'][0]['summary_fields']['direct_access']
assert not direct_access_list[0]['role']['user_capabilities']['unattach']
@pytest.mark.django_db
def test_team_roles_unattach(mocker, team, team_member, inventory, mock_access_method, get):

13
awx/main/tests/functional/api/test_role.py Normal file
View File

@@ -0,0 +1,13 @@
import pytest
from django.core.urlresolvers import reverse
@pytest.mark.django_db
def test_admin_visible_to_orphaned_users(get, alice):
names = set()
response = get(reverse('api:role_list'), user=alice)
for item in response.data['results']:
names.add(item['name'])
assert 'System Auditor' in names
assert 'System Administrator' in names

2
awx/main/tests/functional/api/test_survey_spec.py
View File

@@ -6,7 +6,7 @@ from django.core.urlresolvers import reverse
from awx.main.models.jobs import JobTemplate, Job
from awx.main.models.activity_stream import ActivityStream
from awx.api.license import LicenseForbids
from awx.conf.license import LicenseForbids
from awx.main.access import JobTemplateAccess

4
awx/main/tests/functional/commands/test_cleanup_facts.py
View File

@@ -16,10 +16,10 @@ from awx.main.management.commands.cleanup_facts import CleanupFacts, Command
from awx.main.models.fact import Fact
from awx.main.models.inventory import Host
def mock_feature_enabled(feature, bypass_database=None):
def mock_feature_enabled(feature):
return True
def mock_feature_disabled(feature, bypass_database=None):
def mock_feature_disabled(feature):
return False
@pytest.mark.django_db

66
awx/main/tests/functional/core/test_licenses.py
View File

@@ -1,28 +1,23 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
import json
import mock
import os
import tempfile
import time
import pytest
from datetime import datetime
from awx.main.models import Host
from awx.main.task_engine import TaskSerializer, TaskEngager
from awx.main.task_engine import TaskEnhancer
@pytest.mark.django_db
def test_license_writer(inventory, admin):
writer = TaskEngager(
task_enhancer = TaskEnhancer(
company_name='acmecorp',
contact_name='Michael DeHaan',
contact_email='michael@ansibleworks.com',
license_date=25000, # seconds since epoch
instance_count=500)
data = writer.get_data()
data = task_enhancer.enhance()
Host.objects.bulk_create(
[
@@ -42,13 +37,7 @@ def test_license_writer(inventory, admin):
assert data['license_date'] == 25000
assert data['license_key'] == "11bae31f31c6a6cdcb483a278cdbe98bd8ac5761acd7163a50090b0f098b3a13"
strdata = writer.get_string()
strdata_loaded = json.loads(strdata)
assert strdata_loaded == data
reader = TaskSerializer()
vdata = reader.from_string(strdata)
vdata = task_enhancer.validate_enhancements()
assert vdata['available_instances'] == 500
assert vdata['current_instances'] == 12
@@ -63,70 +52,41 @@ def test_license_writer(inventory, admin):
@pytest.mark.django_db
def test_expired_licenses():
reader = TaskSerializer()
writer = TaskEngager(
task_enhancer = TaskEnhancer(
company_name='Tower',
contact_name='Tower Admin',
contact_email='tower@ansible.com',
license_date=int(time.time() - 3600),
instance_count=100,
trial=True)
strdata = writer.get_string()
vdata = reader.from_string(strdata)
task_enhancer.enhance()
vdata = task_enhancer.validate_enhancements()
assert vdata['compliant'] is False
assert vdata['grace_period_remaining'] < 0
writer = TaskEngager(
task_enhancer = TaskEnhancer(
company_name='Tower',
contact_name='Tower Admin',
contact_email='tower@ansible.com',
license_date=int(time.time() - 2592001),
instance_count=100,
trial=False)
strdata = writer.get_string()
vdata = reader.from_string(strdata)
task_enhancer.enhance()
vdata = task_enhancer.validate_enhancements()
assert vdata['compliant'] is False
assert vdata['grace_period_remaining'] < 0
writer = TaskEngager(
task_enhancer = TaskEnhancer(
company_name='Tower',
contact_name='Tower Admin',
contact_email='tower@ansible.com',
license_date=int(time.time() - 3600),
instance_count=100,
trial=False)
strdata = writer.get_string()
vdata = reader.from_string(strdata)
task_enhancer.enhance()
vdata = task_enhancer.validate_enhancements()
assert vdata['compliant'] is False
assert vdata['grace_period_remaining'] > 0
@pytest.mark.django_db
def test_aws_license():
os.environ['AWX_LICENSE_FILE'] = 'non-existent-license-file.json'
h, path = tempfile.mkstemp()
with os.fdopen(h, 'w') as f:
json.dump({'instance_count': 100}, f)
def fetch_ami(_self):
_self.attributes['ami-id'] = 'ami-00000000'
return True
def fetch_instance(_self):
_self.attributes['instance-id'] = 'i-00000000'
return True
with mock.patch('awx.main.task_engine.TEMPORARY_TASK_FILE', path):
with mock.patch('awx.main.task_engine.TemporaryTaskEngine.fetch_ami', fetch_ami):
with mock.patch('awx.main.task_engine.TemporaryTaskEngine.fetch_instance', fetch_instance):
reader = TaskSerializer()
license = reader.from_file()
assert license['is_aws']
assert license['time_remaining']
assert license['free_instances'] > 0
assert license['grace_period_remaining'] > 0
os.unlink(path)

2
awx/main/tests/functional/test_rbac_api.py
View File

@@ -5,7 +5,7 @@ from django.db import transaction
from django.core.urlresolvers import reverse
from awx.main.models.rbac import Role, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR
def mock_feature_enabled(feature, bypass_database=None):
def mock_feature_enabled(feature):
return True
#@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)

7
awx/main/tests/old/ad_hoc.py
View File

@@ -20,7 +20,6 @@ from crum import impersonate
# AWX
from awx.main.utils import * # noqa
from awx.main.models import * # noqa
from awx.main.conf import tower_settings
from awx.main.tests.base import BaseJobExecutionTest
from awx.main.tests.data.ssh import (
TEST_SSH_KEY_DATA,
@@ -572,14 +571,14 @@ class AdHocCommandApiTest(BaseAdHocCommandTest):
# Try to relaunch ad hoc command when module has been removed from
# allowed list of modules.
try:
ad_hoc_commands = tower_settings.AD_HOC_COMMANDS
tower_settings.AD_HOC_COMMANDS = []
ad_hoc_commands = settings.AD_HOC_COMMANDS
settings.AD_HOC_COMMANDS = []
with self.current_user('admin'):
response = self.get(url, expect=200)
self.assertEqual(response['passwords_needed_to_start'], [])
response = self.post(url, {}, expect=400)
finally:
tower_settings.AD_HOC_COMMANDS = ad_hoc_commands
settings.AD_HOC_COMMANDS = ad_hoc_commands
# Try to relaunch after the inventory has been marked inactive.
self.inventory.delete()

3
awx/main/tests/old/users.py
View File

@@ -15,7 +15,6 @@ from django.test.utils import override_settings
# AWX
from awx.main.models import * # noqa
from awx.main.tests.base import BaseTest
from awx.main.conf import tower_settings
__all__ = ['AuthTokenTimeoutTest', 'AuthTokenLimitTest', 'AuthTokenProxyTest', 'UsersTest', 'LdapTest']
@@ -38,7 +37,7 @@ class AuthTokenTimeoutTest(BaseTest):
response = self._generic_rest(dashboard_url, expect=200, method='get', return_response_object=True, client_kwargs=kwargs)
self.assertIn('Auth-Token-Timeout', response)
self.assertEqual(response['Auth-Token-Timeout'], str(tower_settings.AUTH_TOKEN_EXPIRATION))
self.assertEqual(response['Auth-Token-Timeout'], str(settings.AUTH_TOKEN_EXPIRATION))
class AuthTokenLimitTest(BaseTest):
def setUp(self):

6
awx/main/tests/unit/conftest.py Normal file
View File

@@ -0,0 +1,6 @@
import pytest
@pytest.fixture(autouse=True)
def _disable_database_settings(mocker):
mocker.patch('awx.conf.settings.SettingsWrapper._get_supported_settings', return_value=[])

12
awx/main/tests/unit/commands/test_run_task_system.py → awx/main/tests/unit/scheduler/test_dag.py
View File

@@ -1,10 +1,12 @@
from awx.main.management.commands.run_task_system import (
SimpleDAG,
WorkflowDAG,
)
# Python
import pytest
# AWX
from awx.main.scheduler.dag_simple import SimpleDAG
from awx.main.scheduler.dag_workflow import WorkflowDAG
from awx.main.models import Job
from awx.main.models.workflow import WorkflowJobNode
import pytest
@pytest.fixture
def dag_root():

56
awx/main/tests/unit/test_credentials.py
View File

@@ -1,56 +0,0 @@
from django.core.exceptions import ValidationError
from awx.main.models.credential import validate_ssh_private_key
import pytest
def test_valid_rsa_key():
begin = """-----BEGIN RSA PRIVATE KEY-----"""
end = """-----END RSA PRIVATE KEY-----"""
unvalidated_key = build_key(begin, body, end)
key_data = validate_ssh_private_key(unvalidated_key)
assert key_data['key_type'] == 'rsa'
def test_invalid_key():
unvalidated_key = build_key(key_begin, body, "END KEY")
with pytest.raises(ValidationError):
validate_ssh_private_key(unvalidated_key)
def test_key_type_empty():
unvalidated_key = build_key(key_begin, body, key_end)
key_data = validate_ssh_private_key(unvalidated_key)
assert key_data['key_type'] == 'rsa1'
def build_key(begin, body, end):
return """%s%s%s""" % (begin, body, end)
key_begin = """-----BEGIN PRIVATE KEY-----"""
key_end = """-----END PRIVATE KEY-----"""
body = """
uFZFyag7VVqI+q/oGnQu+wj/pMi5ox+Qz5L3W0D745DzwgDXOeObAfNlr9NtIKbn
sZ5E0+rYB4Q/U0CYr5juNJQV1dbxq2Em1160axboe2QbvX6wE6Sm6wW9b9cr+PoF
MoYQebUnCY0ObrLbrRugSfZc17lyxK0ZGRgPXKhpMg6Ecv8XpvhjUYU9Esyqfuco
/p26Q140/HsHeHYNma0dQHCEjMr/qEzOY1qguHj+hRf3SARtM9Q+YNgpxchcDDVS
O+n+8Ljd/p82bpEJwxmpXealeWbI6gB9/R6wcCL+ZyCZpnHJd/NJ809Vtu47ZdDi
E6jvqS/3AQhuQKhJlLSDIzezB2VKKrHwOvHkg/+uLoCqHN34Gk6Qio7x69SvXy88
a7q9D1l/Zx60o08FyZyqlo7l0l/r8EY+36cuI/lvAvfxc5VHVEOvKseUjFRBiCv9
MkKNxaScoYsPwY7SIS6gD93tg3eM5pA0nfMfya9u1+uq/QCM1gNG3mm6Zd8YG4c/
Dx4bmsj8cp5ni/Ffl/sKzKYq1THunJEFGXOZRibdxk/Fal3SQrRAwy7CgLQL8SMh
IWqcFm25OtSOP1r1LE25t5pQsMdmp0IP2fEF0t/pXPm1ZfrTurPMqpo4FGm2hkki
U3sH/o6nrkSOjklOLWlwtTkkL4dWPlNwc8OYj8zFizXJkAfv1spzhv3lRouNkw4N
Mm22W7us2f3Ob0H5C07k26h6VuXX+0AybD4tIIcUXCLoNTqA0HvqhKpEuHu3Ck10
RaB8xHTxgwdhGVaNHMfy9B9l4tNs3Tb5k0LyeRRGVDhWCFo6axYULYebkj+hFLLY
+JE5RzPDFpTf1xbuT+e56H/lLFCUdDu0bn+D0W4ifXaVFegak4r6O4B53CbMqr+R
t6qDPKLUIuVJXK0J6Ay6XgmheXJGbgKh4OtDsc06gsTCE1nY4f/Z82AQahPBfTtF
J2z+NHdsLPn//HlxspGQtmLpuS7Wx0HYXZ+kPRSiE/vmITw85R2u8JSHQicVNN4C
2rlUo15TIU3tTx+WUIrHKHPidUNNotRb2p9n9FoSidU6upKnQHAT/JNv/zcvaia3
Bhl/wagheWTDnFKSmJ4HlKxplM/32h6MfHqsMVOl4F6eZWKaKgSgN8doXyFJo+sc
yAC6S0gJlD2gQI24iTI4Du1+UGh2MGb69eChvi5mbbdesaZrlR1dRqZpHG+6ob4H
nYLndRvobXS5l6pgGTDRYoUgSbQe21a7Uf3soGl5jHqLWc1zEPwrxV7Wr31mApr6
8VtGZcLSr0691Q1NLO3eIfuhbMN2mssX/Sl4t+4BibaucNIMfmhKQi8uHtwAXb47
+TMFlG2EQhZULFM4fLdF1vaizInU3cBk8lsz8i71tDc+5VQTEwoEB7Gksy/XZWEt
6SGHxXUDtNYa+G2O+sQhgqBjLIkVTV6KJOpvNZM+s8Vzv8qoFnD7isKBBrRvF1bP
GOXEG1jd7nSR0WSwcMCHGOrFEELDQPw3k5jqEdPFgVODoZPr+drZVnVz5SAGBk5Y
wsCNaDW+1dABYFlqRTepP5rrSu9wHnRAZ3ZGv+DHoGqenIC5IBR0sQ==
"""

11
awx/main/tests/unit/test_tasks.py
View File

@@ -10,9 +10,7 @@ from awx.main.tasks import (
send_notifications,
run_administrative_checks,
)
from awx.main.task_engine import TaskSerializer
from awx.main.task_engine import TaskEnhancer
@contextmanager
def apply_patches(_patches):
@@ -51,12 +49,11 @@ def test_send_notifications_list(mocker):
@pytest.mark.parametrize("current_instances,call_count", [(91, 2), (89,1)])
def test_run_admin_checks_usage(mocker, current_instances, call_count):
patches = list()
patches.append(mocker.patch('awx.main.tasks.tower_settings'))
patches.append(mocker.patch('awx.main.tasks.User'))
mock_ts = mocker.Mock(spec=TaskSerializer)
mock_ts.from_database.return_value = {'instance_count': 100, 'current_instances': current_instances}
patches.append(mocker.patch('awx.main.tasks.TaskSerializer', return_value=mock_ts))
mock_te = mocker.Mock(spec=TaskEnhancer)
mock_te.validate_enhancements.return_value = {'instance_count': 100, 'current_instances': current_instances, 'date_warning': True}
patches.append(mocker.patch('awx.main.tasks.TaskEnhancer', return_value=mock_te))
mock_sm = mocker.Mock()
patches.append(mocker.patch('awx.main.tasks.send_mail', wraps=mock_sm))

91
awx/main/tests/unit/test_validators.py Normal file
View File

@@ -0,0 +1,91 @@
from django.core.exceptions import ValidationError
from awx.main.validators import (
validate_private_key,
validate_certificate,
validate_ssh_private_key,
)
from awx.main.tests.data.ssh import (
TEST_SSH_RSA1_KEY_DATA,
TEST_SSH_KEY_DATA,
TEST_SSH_KEY_DATA_LOCKED,
TEST_OPENSSH_KEY_DATA,
TEST_OPENSSH_KEY_DATA_LOCKED,
TEST_SSH_CERT_KEY,
)
import pytest
def test_valid_rsa_key():
valid_key = TEST_SSH_KEY_DATA
pem_objects = validate_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa'
assert not pem_objects[0]['key_enc']
with pytest.raises(ValidationError):
validate_certificate(valid_key)
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa'
assert not pem_objects[0]['key_enc']
def test_valid_locked_rsa_key():
valid_key = TEST_SSH_KEY_DATA_LOCKED
pem_objects = validate_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa'
assert pem_objects[0]['key_enc']
with pytest.raises(ValidationError):
validate_certificate(valid_key)
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa'
assert pem_objects[0]['key_enc']
def test_invalid_rsa_key():
invalid_key = TEST_SSH_KEY_DATA.replace('-----END', '----END')
with pytest.raises(ValidationError):
validate_private_key(invalid_key)
with pytest.raises(ValidationError):
validate_certificate(invalid_key)
with pytest.raises(ValidationError):
validate_ssh_private_key(invalid_key)
def test_valid_openssh_key():
valid_key = TEST_OPENSSH_KEY_DATA
pem_objects = validate_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'ed25519'
assert not pem_objects[0]['key_enc']
with pytest.raises(ValidationError):
validate_certificate(valid_key)
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'ed25519'
assert not pem_objects[0]['key_enc']
def test_valid_locked_openssh_key():
valid_key = TEST_OPENSSH_KEY_DATA_LOCKED
pem_objects = validate_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'ed25519'
assert pem_objects[0]['key_enc']
with pytest.raises(ValidationError):
validate_certificate(valid_key)
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'ed25519'
assert pem_objects[0]['key_enc']
def test_valid_rsa1_key():
valid_key = TEST_SSH_RSA1_KEY_DATA
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa1'
assert not pem_objects[0]['key_enc']
with pytest.raises(ValidationError):
validate_certificate(valid_key)
pem_objects = validate_ssh_private_key(valid_key)
assert pem_objects[0]['key_type'] == 'rsa1'
assert not pem_objects[0]['key_enc']
def test_cert_with_key():
cert_with_key = TEST_SSH_CERT_KEY
with pytest.raises(ValidationError):
validate_private_key(cert_with_key)
with pytest.raises(ValidationError):
validate_certificate(cert_with_key)
pem_objects = validate_ssh_private_key(cert_with_key)
assert pem_objects[0]['type'] == 'CERTIFICATE'
assert pem_objects[1]['key_type'] == 'rsa'
assert not pem_objects[1]['key_enc']

14
awx/main/utils.py
View File

@@ -97,14 +97,14 @@ class RequireDebugTrueOrTest(logging.Filter):
return settings.DEBUG or 'test' in sys.argv
def memoize(ttl=60):
def memoize(ttl=60, cache_key=None):
'''
Decorator to wrap a function and cache its result.
'''
from django.core.cache import cache
def _memoizer(f, *args, **kwargs):
key = slugify('%s %r %r' % (f.__name__, args, kwargs))
key = cache_key or slugify('%s %r %r' % (f.__name__, args, kwargs))
value = cache.get(key)
if value is None:
value = f(*args, **kwargs)
@@ -475,6 +475,7 @@ def cache_list_capabilities(page, prefetch_list, model, user):
obj.capabilities_cache[display_method] = True
@memoize()
def get_system_task_capacity():
'''
Measure system memory and use it as a baseline for determining the system's capacity
@@ -537,8 +538,8 @@ def build_proot_temp_dir():
'''
Create a temporary directory for proot to use.
'''
from awx.main.conf import tower_settings
path = tempfile.mkdtemp(prefix='ansible_tower_proot_', dir=tower_settings.AWX_PROOT_BASE_PATH)
from django.conf import settings
path = tempfile.mkdtemp(prefix='ansible_tower_proot_', dir=settings.AWX_PROOT_BASE_PATH)
os.chmod(path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
return path
@@ -551,14 +552,13 @@ def wrap_args_with_proot(args, cwd, **kwargs):
- /var/log/supervisor
- /tmp (except for own tmp files)
'''
from awx.main.conf import tower_settings
from django.conf import settings
new_args = [getattr(settings, 'AWX_PROOT_CMD', 'proot'), '-v',
str(getattr(settings, 'AWX_PROOT_VERBOSITY', '0')), '-r', '/']
hide_paths = ['/etc/tower', '/var/lib/awx', '/var/log',
tempfile.gettempdir(), settings.PROJECTS_ROOT,
settings.JOBOUTPUT_ROOT]
hide_paths.extend(getattr(tower_settings, 'AWX_PROOT_HIDE_PATHS', None) or [])
hide_paths.extend(getattr(settings, 'AWX_PROOT_HIDE_PATHS', None) or [])
for path in sorted(set(hide_paths)):
if not os.path.exists(path):
continue
@@ -578,7 +578,7 @@ def wrap_args_with_proot(args, cwd, **kwargs):
show_paths.append(settings.ANSIBLE_VENV_PATH)
if settings.TOWER_USE_VENV:
show_paths.append(settings.TOWER_VENV_PATH)
show_paths.extend(getattr(tower_settings, 'AWX_PROOT_SHOW_PATHS', None) or [])
show_paths.extend(getattr(settings, 'AWX_PROOT_SHOW_PATHS', None) or [])
for path in sorted(set(show_paths)):
if not os.path.exists(path):
continue

168
awx/main/validators.py Normal file
View File

@@ -0,0 +1,168 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
# Python
import base64
import re
# Django
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import ValidationError
def validate_pem(data, min_keys=0, max_keys=None, min_certs=0, max_certs=None):
"""
Validate the given PEM data is valid and contains the required numbers of
keys and certificates.
Return a list of PEM objects, where each object is a dict with the following
keys:
- 'all': The entire string for the PEM object including BEGIN/END lines.
- 'type': The type of PEM object ('PRIVATE KEY' or 'CERTIFICATE').
- 'data': The string inside the BEGIN/END lines.
- 'b64': Key/certificate as a base64-encoded string.
- 'bin': Key/certificate as bytes.
- 'key_type': Only when type == 'PRIVATE KEY', one of 'rsa', 'dsa',
'ecdsa', 'ed25519' or 'rsa1'.
- 'key_enc': Only when type == 'PRIVATE KEY', boolean indicating if key is
encrypted.
"""
# Map the X in BEGIN X PRIVATE KEY to the key type (ssh-keygen -t).
# Tower jobs using OPENSSH format private keys may still fail if the
# system SSH implementation lacks support for this format.
private_key_types = {
'RSA': 'rsa',
'DSA': 'dsa',
'EC': 'ecdsa',
'OPENSSH': 'ed25519',
'': 'rsa1',
}
# Build regular expressions for matching each object in the PEM file.
pem_obj_re = re.compile(
r'^(-{4,}) *BEGIN ([A-Z ]+?) *\1[\r\n]+' +
r'(.+?)[\r\n]+\1 *END \2 *\1[\r\n]?(.*?)$', re.DOTALL,
)
pem_obj_header_re = re.compile(r'^(.+?):\s*?(.+?)(\\??)$')
pem_objects = []
key_count, cert_count = 0, 0
data = data.lstrip()
while data:
match = pem_obj_re.match(data)
if not match:
raise ValidationError(_('Invalid certificate or key: %r...') % data[:100])
data = match.group(4).lstrip()
# Check PEM object type, check key type if private key.
pem_obj_info = {}
pem_obj_info['all'] = match.group(0)
pem_obj_info['type'] = pem_obj_type = match.group(2)
if pem_obj_type.endswith('PRIVATE KEY'):
key_count += 1
pem_obj_info['type'] = 'PRIVATE KEY'
key_type = pem_obj_type.replace('PRIVATE KEY', '').strip()
try:
pem_obj_info['key_type'] = private_key_types[key_type]
except KeyError:
raise ValidationError(_('Invalid private key: unsupported type "%s"') % key_type)
elif pem_obj_type == 'CERTIFICATE':
cert_count += 1
else:
raise ValidationError(_('Unsupported PEM object type: "%s"') % pem_obj_type)
# Ensure that this PEM object is valid base64 data.
pem_obj_info['data'] = match.group(3)
base64_data = ''
line_continues = False
for line in pem_obj_info['data'].splitlines():
line = line.strip()
if not line:
continue
if line_continues:
line_continues = line.endswith('\\')
continue
line_match = pem_obj_header_re.match(line)
if line_match:
line_continues = line.endswith('\\')
continue
base64_data += line
try:
decoded_data = base64.b64decode(base64_data)
if not decoded_data:
raise TypeError
pem_obj_info['b64'] = base64_data
pem_obj_info['bin'] = decoded_data
except TypeError:
raise ValidationError(_('Invalid base64-encoded data'))
# If private key, check whether it is encrypted.
if pem_obj_info.get('key_type', '') == 'ed25519':
# See https://github.com/openssh/openssh-portable/blob/master/sshkey.c#L3218
# Decoded key data starts with magic string (null-terminated), four byte
# length field, followed by the ciphername -- if ciphername is anything
# other than 'none' the key is encrypted.
pem_obj_info['key_enc'] = not bool(pem_obj_info['bin'].startswith('openssh-key-v1\x00\x00\x00\x00\x04none'))
elif pem_obj_info.get('key_type', ''):
pem_obj_info['key_enc'] = bool('ENCRYPTED' in pem_obj_info['data'])
pem_objects.append(pem_obj_info)
# Validate that the number of keys and certs provided are within the limits.
key_count_dict = dict(min_keys=min_keys, max_keys=max_keys, key_count=key_count)
if key_count < min_keys:
if min_keys == 1:
if max_keys == min_keys:
raise ValidationError(_('Exactly one private key is required.'))
else:
raise ValidationError(_('At least one private key is required.'))
else:
raise ValidationError(_('At least %(min_keys)d private keys are required, only %(key_count)d provided.') % key_count_dict)
elif max_keys is not None and key_count > max_keys:
if max_keys == 1:
raise ValidationError(_('Only one private key is allowed, %(key_count)d provided.') % key_count_dict)
else:
raise ValidationError(_('No more than %(max_keys)d private keys are allowed, %(key_count)d provided.') % key_count_dict)
cert_count_dict = dict(min_certs=min_certs, max_certs=max_certs, cert_count=cert_count)
if cert_count < min_certs:
if min_certs == 1:
if max_certs == min_certs:
raise ValidationError(_('Exactly one certificate is required.'))
else:
raise ValidationError(_('At least one certificate is required.'))
else:
raise ValidationError(_('At least %(min_certs)d certificates are required, only %(cert_count)d provided.') % cert_count_dict)
elif max_certs is not None and cert_count > max_certs:
if max_certs == 1:
raise ValidationError(_('Only one certificate is allowed, %(cert_count)d provided.') % cert_count_dict)
else:
raise ValidationError(_('No more than %(max_certs)d certificates are allowed, %(cert_count)d provided.') % cert_count_dict)
return pem_objects
def validate_private_key(data):
"""
Validate that data contains exactly one private key.
"""
return validate_pem(data, min_keys=1, max_keys=1, max_certs=0)
def validate_certificate(data):
"""
Validate that data contains one or more certificates. Adds BEGIN/END lines
if necessary.
"""
if 'BEGIN CERTIFICATE' not in data:
data = '-----BEGIN CERTIFICATE-----\n{}\n-----END CERTIFICATE-----\n'.format(data)
return validate_pem(data, max_keys=0, min_certs=1)
def validate_ssh_private_key(data):
"""
Validate that data contains at least one private key and optionally
certificates; should handle any valid options for ssh_private_key on a
credential.
"""
return validate_pem(data, min_keys=1)