Merge branch 'release_3.0.2' into 3081

This commit is contained in:
Akita Noek 2016-08-16 13:52:57 -04:00
commit db87e3cb30
23 changed files with 252 additions and 73 deletions

View File

@ -1983,7 +1983,7 @@ class JobSerializer(UnifiedJobSerializer, JobOptionsSerializer):
return ret
if 'job_template' in ret and not obj.job_template:
ret['job_template'] = None
if obj.job_template and obj.job_template.survey_enabled and 'extra_vars' in ret:
if 'extra_vars' in ret:
ret['extra_vars'] = obj.display_extra_vars()
return ret

View File

@ -1,4 +1,3 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
@ -2987,7 +2986,7 @@ class JobJobTasksList(BaseJobEventsList):
# need stats on grandchildren, sorted by child.
queryset = (JobEvent.objects.filter(parent__parent=parent_task,
parent__event__in=STARTING_EVENTS)
.values('parent__id', 'event', 'changed')
.values('parent__id', 'event', 'changed', 'failed')
.annotate(num=Count('event'))
.order_by('parent__id'))
@ -3048,10 +3047,13 @@ class JobJobTasksList(BaseJobEventsList):
# make appropriate changes to the task data.
for child_data in data.get(task_start_event.id, []):
if child_data['event'] == 'runner_on_failed':
task_data['failed'] = True
task_data['host_count'] += child_data['num']
task_data['reported_hosts'] += child_data['num']
task_data['failed_count'] += child_data['num']
if child_data['failed']:
task_data['failed'] = True
task_data['failed_count'] += child_data['num']
else:
task_data['skipped_count'] += child_data['num']
elif child_data['event'] == 'runner_on_ok':
task_data['host_count'] += child_data['num']
task_data['reported_hosts'] += child_data['num']

View File

@ -0,0 +1,20 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations, models
import jsonfield.fields
class Migration(migrations.Migration):
dependencies = [
('main', '0029_v302_add_ask_skip_tags'),
]
operations = [
migrations.AddField(
model_name='job',
name='survey_passwords',
field=jsonfield.fields.JSONField(default={}, editable=False, blank=True),
),
]

View File

@ -0,0 +1,18 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from awx.main.migrations import _save_password_keys
from awx.main.migrations import _migration_utils as migration_utils
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0030_v302_job_survey_passwords'),
]
operations = [
migrations.RunPython(migration_utils.set_current_apps_for_migrations),
migrations.RunPython(_save_password_keys.migrate_survey_passwords),
]

View File

@ -0,0 +1,27 @@
def survey_password_variables(survey_spec):
vars = []
# Get variables that are type password
if 'spec' not in survey_spec:
return vars
for survey_element in survey_spec['spec']:
if 'type' in survey_element and survey_element['type'] == 'password':
vars.append(survey_element['variable'])
return vars
def migrate_survey_passwords(apps, schema_editor):
'''Take the output of the Job Template password list for all that
have a survey enabled, and then save it into the job model.
'''
Job = apps.get_model('main', 'Job')
for job in Job.objects.iterator():
if not job.job_template:
continue
jt = job.job_template
if jt.survey_spec is not None and jt.survey_enabled:
password_list = survey_password_variables(jt.survey_spec)
hide_password_dict = {}
for password in password_list:
hide_password_dict[password] = "$encrypted$"
job.survey_passwords = hide_password_dict
job.save()

View File

@ -27,7 +27,7 @@ from awx.main.models.unified_jobs import * # noqa
from awx.main.models.notifications import NotificationTemplate
from awx.main.utils import decrypt_field, ignore_inventory_computed_fields
from awx.main.utils import emit_websocket_notification
from awx.main.redact import PlainTextCleaner, REPLACE_STR
from awx.main.redact import PlainTextCleaner
from awx.main.conf import tower_settings
from awx.main.fields import ImplicitRoleField
from awx.main.models.mixins import ResourceMixin
@ -249,7 +249,7 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, ResourceMixin):
'playbook', 'credential', 'cloud_credential', 'network_credential', 'forks', 'schedule',
'limit', 'verbosity', 'job_tags', 'extra_vars', 'launch_type',
'force_handlers', 'skip_tags', 'start_at_task', 'become_enabled',
'labels',]
'labels', 'survey_passwords']
def resource_validation_data(self):
'''
@ -447,11 +447,21 @@ class JobTemplate(UnifiedJobTemplate, JobOptions, ResourceMixin):
if field == 'extra_vars' and self.survey_enabled and self.survey_spec:
# Accept vars defined in the survey and no others
survey_vars = [question['variable'] for question in self.survey_spec.get('spec', [])]
for key in kwargs[field]:
extra_vars = kwargs[field]
if isinstance(extra_vars, basestring):
try:
extra_vars = json.loads(extra_vars)
except (ValueError, TypeError):
try:
extra_vars = yaml.safe_load(extra_vars)
assert isinstance(extra_vars, dict)
except (yaml.YAMLError, TypeError, AttributeError, AssertionError):
extra_vars = {}
for key in extra_vars:
if key in survey_vars:
prompted_fields[field][key] = kwargs[field][key]
prompted_fields[field][key] = extra_vars[key]
else:
ignored_fields[field][key] = kwargs[field][key]
ignored_fields[field][key] = extra_vars[key]
else:
ignored_fields[field] = kwargs[field]
@ -514,6 +524,11 @@ class Job(UnifiedJob, JobOptions):
editable=False,
through='JobHostSummary',
)
survey_passwords = JSONField(
blank=True,
default={},
editable=False,
)
@classmethod
def _get_parent_field_name(cls):
@ -730,16 +745,12 @@ class Job(UnifiedJob, JobOptions):
'''
Hides fields marked as passwords in survey.
'''
if self.extra_vars and self.job_template and self.job_template.survey_enabled:
try:
extra_vars = json.loads(self.extra_vars)
for key in self.job_template.survey_password_variables():
if key in extra_vars:
extra_vars[key] = REPLACE_STR
return json.dumps(extra_vars)
except ValueError:
pass
return self.extra_vars
if self.survey_passwords:
extra_vars = json.loads(self.extra_vars)
extra_vars.update(self.survey_passwords)
return json.dumps(extra_vars)
else:
return self.extra_vars
def _survey_search_and_replace(self, content):
# Use job template survey spec to identify password fields.

View File

@ -32,7 +32,7 @@ from djcelery.models import TaskMeta
from awx.main.models.base import * # noqa
from awx.main.models.schedules import Schedule
from awx.main.utils import decrypt_field, emit_websocket_notification, _inventory_updates
from awx.main.redact import UriCleaner
from awx.main.redact import UriCleaner, REPLACE_STR
__all__ = ['UnifiedJobTemplate', 'UnifiedJob']
@ -343,6 +343,14 @@ class UnifiedJobTemplate(PolymorphicModel, CommonModelNameNotUnique, Notificatio
create_kwargs[field_name] = getattr(self, field_name)
new_kwargs = self._update_unified_job_kwargs(**create_kwargs)
unified_job = unified_job_class(**new_kwargs)
# For JobTemplate-based jobs with surveys, save list for perma-redaction
if (hasattr(self, 'survey_spec') and getattr(self, 'survey_enabled', False) and
not getattr(unified_job, 'survey_passwords', False)):
password_list = self.survey_password_variables()
hide_password_dict = {}
for password in password_list:
hide_password_dict[password] = REPLACE_STR
unified_job.survey_passwords = hide_password_dict
unified_job.save()
for field_name, src_field_value in m2m_fields.iteritems():
dest_field = getattr(unified_job, field_name)

View File

@ -26,16 +26,16 @@ def survey_spec_factory():
return create_survey_spec
@pytest.fixture
def job_with_secret_key_factory(job_template_factory):
def job_template_with_survey_passwords_factory(job_template_factory):
def rf(persisted):
"Returns job with linked JT survey with password survey questions"
objects = job_template_factory('jt', organization='org1', survey=[
{'variable': 'submitter_email', 'type': 'text', 'default': 'foobar@redhat.com'},
{'variable': 'secret_key', 'default': '6kQngg3h8lgiSTvIEb21', 'type': 'password'},
{'variable': 'SSN', 'type': 'password'}], jobs=[1], persisted=persisted)
return objects.jobs[1]
{'variable': 'SSN', 'type': 'password'}], persisted=persisted)
return objects.job_template
return rf
@pytest.fixture
def job_with_secret_key_unit(job_with_secret_key_factory):
return job_with_secret_key_factory(persisted=False)
def job_template_with_survey_passwords_unit(job_template_with_survey_passwords_factory):
return job_template_with_survey_passwords_factory(persisted=False)

View File

@ -3,12 +3,14 @@ import mock
# AWX
from awx.api.serializers import JobTemplateSerializer, JobLaunchSerializer
from awx.main.models.jobs import JobTemplate
from awx.main.models.jobs import JobTemplate, Job
from awx.main.models.projects import ProjectOptions
from awx.main.migrations import _save_password_keys as save_password_keys
# Django
from django.test.client import RequestFactory
from django.core.urlresolvers import reverse
from django.apps import apps
@property
def project_playbooks(self):
@ -348,3 +350,20 @@ def test_disallow_template_delete_on_running_job(job_template_factory, delete, a
objects.job_template.create_unified_job()
delete_response = delete(reverse('api:job_template_detail', args=[objects.job_template.pk]), user=admin_user)
assert delete_response.status_code == 409
@pytest.mark.django_db
def test_save_survey_passwords_to_job(job_template_with_survey_passwords):
"""Test that when a new job is created, the survey_passwords field is
given all of the passwords that exist in the JT survey"""
job = job_template_with_survey_passwords.create_unified_job()
assert job.survey_passwords == {'SSN': '$encrypted$', 'secret_key': '$encrypted$'}
@pytest.mark.django_db
def test_save_survey_passwords_on_migration(job_template_with_survey_passwords):
"""Test that when upgrading to 3.0.2, the jobs connected to a JT that has
a survey with passwords in it, the survey passwords get saved to the
job survey_passwords field."""
Job.objects.create(job_template=job_template_with_survey_passwords)
save_password_keys.migrate_survey_passwords(apps, None)
job = job_template_with_survey_passwords.jobs.all()[0]
assert job.survey_passwords == {'SSN': '$encrypted$', 'secret_key': '$encrypted$'}

View File

@ -193,7 +193,8 @@ def test_launch_with_non_empty_survey_spec_no_license(job_template_factory, post
@pytest.mark.django_db
@pytest.mark.survey
def test_redact_survey_passwords_in_activity_stream(job_with_secret_key):
def test_redact_survey_passwords_in_activity_stream(job_template_with_survey_passwords):
job_template_with_survey_passwords.create_unified_job()
AS_record = ActivityStream.objects.filter(object1='job').all()[0]
changes_dict = json.loads(AS_record.changes)
extra_vars = json.loads(changes_dict['extra_vars'])

View File

@ -206,8 +206,8 @@ def notification(notification_template):
subject='email subject')
@pytest.fixture
def job_with_secret_key(job_with_secret_key_factory):
return job_with_secret_key_factory(persisted=True)
def job_template_with_survey_passwords(job_template_with_survey_passwords_factory):
return job_template_with_survey_passwords_factory(persisted=True)
@pytest.fixture
def admin(user):

View File

@ -1,4 +1,5 @@
import pytest
import json
def test_missing_project_error(job_template_factory):
@ -34,7 +35,18 @@ def test_inventory_credential_contradictions(job_template_factory):
assert 'inventory' in validation_errors
assert 'credential' in validation_errors
def test_survey_answers_as_string(job_template_factory):
objects = job_template_factory(
'job-template-with-survey',
survey=['var1'],
persisted=False)
jt = objects.job_template
user_extra_vars = json.dumps({'var1': 'asdf'})
accepted, ignored = jt._accept_or_ignore_job_kwargs(extra_vars=user_extra_vars)
assert 'var1' in accepted['extra_vars']
@pytest.mark.survey
def test_survey_password_list(job_with_secret_key_unit):
"""Verify that survey_password_variables method gives a list of survey passwords"""
assert job_with_secret_key_unit.job_template.survey_password_variables() == ['secret_key', 'SSN']
def test_job_template_survey_password_redaction(job_template_with_survey_passwords_unit):
"""Tests the JobTemplate model's funciton to redact passwords from
extra_vars - used when creating a new job"""
assert job_template_with_survey_passwords_unit.survey_password_variables() == ['secret_key', 'SSN']

View File

@ -2,6 +2,7 @@ import pytest
import json
from awx.main.tasks import RunJob
from awx.main.models import Job
@pytest.fixture
@ -14,9 +15,19 @@ def job(mocker):
'launch_type': 'manual'})
@pytest.mark.survey
def test_job_redacted_extra_vars(job_with_secret_key_unit):
"""Verify that this method redacts vars marked as passwords in a survey"""
assert json.loads(job_with_secret_key_unit.display_extra_vars()) == {
def test_job_survey_password_redaction():
"""Tests the Job model's funciton to redact passwords from
extra_vars - used when displaying job information"""
job = Job(
name="test-job-with-passwords",
extra_vars=json.dumps({
'submitter_email': 'foobar@redhat.com',
'secret_key': '6kQngg3h8lgiSTvIEb21',
'SSN': '123-45-6789'}),
survey_passwords={
'secret_key': '$encrypted$',
'SSN': '$encrypted$'})
assert json.loads(job.display_extra_vars()) == {
'submitter_email': 'foobar@redhat.com',
'secret_key': '$encrypted$',
'SSN': '$encrypted$'}

View File

@ -18,10 +18,11 @@ import json
# http://urllib3.readthedocs.org/en/latest/security.html#disabling-warnings
requests.packages.urllib3.disable_warnings()
class CloudFormsInventory(object):
def _empty_inventory(self):
return {"_meta" : {"hostvars" : {}}}
return {"_meta": {"hostvars": {}}}
def __init__(self):
''' Main execution path '''
@ -43,7 +44,7 @@ class CloudFormsInventory(object):
# This doesn't exist yet and needs to be added
if self.args.host:
data2 = { }
data2 = {}
print json.dumps(data2, indent=2)
def parse_cli_args(self):
@ -51,9 +52,9 @@ class CloudFormsInventory(object):
parser = argparse.ArgumentParser(description='Produce an Ansible Inventory file based on CloudForms')
parser.add_argument('--list', action='store_true', default=False,
help='List instances (default: False)')
help='List instances (default: False)')
parser.add_argument('--host', action='store',
help='Get all the variables about a specific instance')
help='Get all the variables about a specific instance')
self.args = parser.parse_args()
def read_settings(self):
@ -97,30 +98,47 @@ class CloudFormsInventory(object):
def get_hosts(self):
''' Gets host from CloudForms '''
r = requests.get("https://" + self.cloudforms_hostname + "/api/vms?expand=resources&attributes=name,power_state", auth=(self.cloudforms_username,self.cloudforms_password), verify=False)
r = requests.get("https://{0}/api/vms?expand=resources&attributes=all".format(self.cloudforms_hostname),
auth=(self.cloudforms_username, self.cloudforms_password), verify=False)
obj = r.json()
#Remove objects that don't matter
del obj["count"]
del obj["subcount"]
del obj["name"]
# Create groups+hosts based on host data
for resource in obj.get('resources', []):
#Create a new list to grab VMs with power_state on to add to a new list
#I'm sure there is a cleaner way to do this
newlist = []
getnext = False
for x in obj.items():
for y in x[1]:
for z in y.items():
if getnext == True:
newlist.append(z[1])
getnext = False
if ( z[0] == "power_state" and z[1] == "on" ):
getnext = True
newdict = {'hosts': newlist}
newdict2 = {'Dynamic_CloudForms': newdict}
print json.dumps(newdict2, indent=2)
# Maintain backwards compat by creating `Dynamic_CloudForms` group
if 'Dynamic_CloudForms' not in self.inventory:
self.inventory['Dynamic_CloudForms'] = []
self.inventory['Dynamic_CloudForms'].append(resource['name'])
# Add host to desired groups
for key in ('vendor', 'type', 'location'):
if key in resource:
# Create top-level group
if key not in self.inventory:
self.inventory[key] = dict(children=[], vars={}, hosts=[])
# if resource['name'] not in self.inventory[key]['hosts']:
# self.inventory[key]['hosts'].append(resource['name'])
# Create sub-group
if resource[key] not in self.inventory:
self.inventory[resource[key]] = dict(children=[], vars={}, hosts=[])
# self.inventory[resource[key]]['hosts'].append(resource['name'])
# Add sub-group, as a child of top-level
if resource[key] not in self.inventory[key]['children']:
self.inventory[key]['children'].append(resource[key])
# Add host to sub-group
if resource['name'] not in self.inventory[resource[key]]:
self.inventory[resource[key]]['hosts'].append(resource['name'])
# Delete 'actions' key
del resource['actions']
# Add _meta hostvars
self.inventory['_meta']['hostvars'][resource['name']] = resource
print json.dumps(self.inventory, indent=2)
# Run the script
CloudFormsInventory()

View File

@ -682,6 +682,16 @@ SATELLITE6_HOST_FILTER = r'^.+$'
SATELLITE6_EXCLUDE_EMPTY_GROUPS = True
SATELLITE6_INSTANCE_ID_VAR = 'foreman.id'
# ---------------------
# ----- CloudForms -----
# ---------------------
CLOUDFORMS_ENABLED_VAR = 'power_state'
CLOUDFORMS_ENABLED_VALUE = 'on'
CLOUDFORMS_GROUP_FILTER = r'^.+$'
CLOUDFORMS_HOST_FILTER = r'^.+$'
CLOUDFORMS_EXCLUDE_EMPTY_GROUPS = True
CLOUDFORMS_INSTANCE_ID_VAR = 'id'
# ---------------------
# -- Activity Stream --
# ---------------------

View File

@ -46,13 +46,13 @@ export function CredentialsList($scope, $rootScope, $location, $log,
Wait('stop');
$('#prompt-modal').modal('hide');
list.fields.kind.searchOptions = $scope.credential_kind_options;
list.fields.kind.searchOptions = $scope.credential_kind_options_list;
// Translate the kind value
for (i = 0; i < $scope.credentials.length; i++) {
for (j = 0; j < $scope.credential_kind_options.length; j++) {
if ($scope.credential_kind_options[j].value === $scope.credentials[i].kind) {
$scope.credentials[i].kind = $scope.credential_kind_options[j].label;
for (j = 0; j < $scope.credential_kind_options_list.length; j++) {
if ($scope.credential_kind_options_list[j].value === $scope.credentials[i].kind) {
$scope.credentials[i].kind = $scope.credential_kind_options_list[j].label;
break;
}
}
@ -77,6 +77,15 @@ export function CredentialsList($scope, $rootScope, $location, $log,
$scope.search(list.iterator);
});
// Load the list of options for Kind
GetChoices({
scope: $scope,
url: defaultUrl,
field: 'kind',
variable: 'credential_kind_options_list',
callback: 'choicesReadyCredential'
});
$scope.addCredential = function () {
$state.transitionTo('credentials.add');
};

View File

@ -100,6 +100,7 @@
// equal to case 'ec2' || 'rax' || 'azure' || 'azure_rm' || 'vmware' || 'satellite6' || 'cloudforms' || 'openstack'
else{
var credentialBasePath = (source === 'ec2') ? GetBasePath('credentials') + '?kind=aws' : GetBasePath('credentials') + (source === '' ? '' : '?kind=' + (source));
$scope.cloudCredentialRequired = source !== '' && source !== 'custom' && source !== 'ec2' ? true : false;
CredentialList.basePath = credentialBasePath;
LookUpInit({
scope: $scope,
@ -122,7 +123,7 @@
$scope.group_by_choices = source === 'ec2' ? $scope.ec2_group_by : null;
// azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint
$scope.source_region_choices = source === 'azure_rm' ? $scope.azure_regions : $scope[source + '_regions'];
$scope.cloudCredentialRequired = source !== '' && source !== 'custom' ? true : false;
$scope.cloudCredentialRequired = source !== '' && source !== 'custom' && source !== 'ec2' ? true : false;
$scope.group_by = null;
$scope.source_regions = null;
$scope.credential = null;

View File

@ -100,6 +100,7 @@
else{
var credentialBasePath = (source.value === 'ec2') ? GetBasePath('credentials') + '?kind=aws' : GetBasePath('credentials') + (source.value === '' ? '' : '?kind=' + (source.value));
CredentialList.basePath = credentialBasePath;
$scope.cloudCredentialRequired = source.value !== '' && source.value !== 'custom' && source.value !== 'ec2' ? true : false;
LookUpInit({
scope: $scope,
url: credentialBasePath,
@ -122,7 +123,7 @@
// reset fields
// azure_rm regions choices are keyed as "azure" in an OPTIONS request to the inventory_sources endpoint
$scope.source_region_choices = source.value === 'azure_rm' ? $scope.azure_regions : $scope[source.value + '_regions'];
$scope.cloudCredentialRequired = source.value !== '' && source.value !== 'custom' ? true : false;
$scope.cloudCredentialRequired = source.value !== '' && source.value !== 'custom' && source.value !== 'ec2' ? true : false;
$scope.group_by = null;
$scope.source_regions = null;
$scope.credential = null;

View File

@ -149,6 +149,12 @@ export default
if(field.type === 'number'){
$scope[i] = Number($scope[i]);
}
if(field.name === "username" && $scope.notification_type.value === "email" && value === null){
$scope[i] = "";
}
if(field.type === 'sensitive' && value === null){
$scope[i] = "";
}
return $scope[i];
}

View File

@ -223,6 +223,12 @@ export default
if(field.type === 'number'){
$scope[i] = Number($scope[i]);
}
if(field.name === "username" && $scope.notification_type.value === "email" && value === null){
$scope[i] = "";
}
if(field.type === 'sensitive' && value === null){
$scope[i] = "";
}
return $scope[i];
}

View File

@ -59,10 +59,6 @@ export default function() {
username: {
label: 'Username',
type: 'text',
awRequiredWhen: {
reqExpression: "email_required",
init: "false"
},
ngShow: "notification_type.value == 'email' ",
subForm: 'typeSubForm'
},

View File

@ -28,7 +28,7 @@ function () {
obj.passwordLabel = ' Password';
obj.email_required = true;
obj.port_required = true;
obj.password_required = true;
obj.password_required = false;
break;
case 'slack':
obj.tokenLabel =' Token';

View File

@ -14,8 +14,11 @@ attempt=0
while [[ $attempt -lt $retry_attempts ]]
do
status_code=`curl -s -i --data "host_config_key=$2" http://$1/api/v1/job_templates/$3/callback/ | head -n 1 | awk '{print $2}'`
if [[ $status_code == 202 ]]
if [[ $status_code -ge 300 ]]
then
echo "${status_code} received, encountered problem, halting."
exit 1
else
exit 0
fi
attempt=$(( attempt + 1 ))