diff --git a/.gitignore b/.gitignore
index 57e2baf042..48c5934377 100644
--- a/.gitignore
+++ b/.gitignore
@@ -108,6 +108,7 @@ reports
*.results
local/
*.mo
+requirements/vendor
# AWX python libs populated by requirements.txt
awx/lib/.deps_built
diff --git a/Makefile b/Makefile
index 0d933cff61..3d378c60c9 100644
--- a/Makefile
+++ b/Makefile
@@ -225,15 +225,18 @@ clean-tmp:
clean-venv:
rm -rf venv/
+clean-dist:
+ rm -rf dist
+
# Remove temporary build files, compiled Python files.
-clean: clean-rpm clean-deb clean-ui clean-tar clean-packer clean-bundle
+clean: clean-rpm clean-deb clean-ui clean-tar clean-packer clean-bundle clean-dist
rm -rf awx/public
rm -rf awx/lib/site-packages
- rm -rf dist/*
rm -rf awx/job_status
rm -rf awx/job_output
rm -rf reports
rm -f awx/awx_test.sqlite3
+ rm -rf requirements/vendor
rm -rf tmp
mkdir tmp
rm -rf build $(NAME)-$(VERSION) *.egg-info
@@ -282,7 +285,11 @@ virtualenv_tower:
fi
requirements_ansible: virtualenv_ansible
- $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements_ansible.txt
+ if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
+ cat requirements/requirements_ansible.txt requirements/requirements_ansible_local.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --ignore-installed -r /dev/stdin ; \
+ else \
+ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | $(VENV_BASE)/ansible/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
+ fi
$(VENV_BASE)/ansible/bin/pip uninstall --yes -r requirements/requirements_ansible_uninstall.txt
requirements_ansible_dev:
@@ -292,7 +299,11 @@ requirements_ansible_dev:
# Install third-party requirements needed for Tower's environment.
requirements_tower: virtualenv_tower
- $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed --no-binary $(SRC_ONLY_PKGS) -r requirements/requirements.txt
+ if [[ "$(PIP_OPTIONS)" == *"--no-index"* ]]; then \
+ cat requirements/requirements.txt requirements/requirements_local.txt | $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --ignore-installed -r /dev/stdin ; \
+ else \
+ cat requirements/requirements.txt requirements/requirements_git.txt | $(VENV_BASE)/tower/bin/pip install $(PIP_OPTIONS) --no-binary $(SRC_ONLY_PKGS) --ignore-installed -r /dev/stdin ; \
+ fi
$(VENV_BASE)/tower/bin/pip uninstall --yes -r requirements/requirements_tower_uninstall.txt
requirements_tower_dev:
@@ -690,39 +701,44 @@ setup_bundle_tarball: setup-bundle-build setup-bundle-build/$(OFFLINE_TAR_FILE)
rpm-build:
mkdir -p $@
-rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE)
+rpm-build/$(SDIST_TAR_FILE): rpm-build dist/$(SDIST_TAR_FILE) tar-build/$(SETUP_TAR_FILE)
cp packaging/rpm/$(NAME).spec rpm-build/
cp packaging/rpm/tower.te rpm-build/
cp packaging/rpm/tower.fc rpm-build/
cp packaging/rpm/$(NAME).sysconfig rpm-build/
cp packaging/remove_tower_source.py rpm-build/
cp packaging/bytecompile.sh rpm-build/
+ cp tar-build/$(SETUP_TAR_FILE) rpm-build/
if [ "$(OFFICIAL)" != "yes" ] ; then \
(cd dist/ && tar zxf $(SDIST_TAR_FILE)) ; \
(cd dist/ && mv $(NAME)-$(VERSION)-$(BUILD) $(NAME)-$(VERSION)) ; \
(cd dist/ && tar czf ../rpm-build/$(SDIST_TAR_FILE) $(NAME)-$(VERSION)) ; \
ln -sf $(SDIST_TAR_FILE) rpm-build/$(NAME)-$(VERSION).tar.gz ; \
+ (cd tar-build/ && tar zxf $(SETUP_TAR_FILE)) ; \
+ (cd tar-build/ && mv $(NAME)-setup-$(VERSION)-$(BUILD) $(NAME)-setup-$(VERSION)) ; \
+ (cd tar-build/ && tar czf ../rpm-build/$(SETUP_TAR_FILE) $(NAME)-setup-$(VERSION)) ; \
+ ln -sf $(SETUP_TAR_FILE) rpm-build/$(NAME)-setup-$(VERSION).tar.gz ; \
else \
cp -a dist/$(SDIST_TAR_FILE) rpm-build/ ; \
fi
rpmtar: sdist rpm-build/$(SDIST_TAR_FILE)
-brewrpmtar: rpm-build/python-deps.tar.gz rpmtar
+brewrpmtar: rpm-build/python-deps.tar.gz requirements/requirements_local.txt requirements/requirements_ansible_local.txt rpmtar
rpm-build/python-deps.tar.gz: requirements/vendor rpm-build
tar czf rpm-build/python-deps.tar.gz requirements/vendor
requirements/vendor:
- pip download \
+ cat requirements/requirements.txt requirements/requirements_git.txt | pip download \
--no-binary=:all: \
- --requirement=requirements/requirements_ansible.txt \
+ --requirement=/dev/stdin \
--dest=$@ \
--exists-action=i
- pip download \
+ cat requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt | pip download \
--no-binary=:all: \
- --requirement=requirements/requirements.txt \
+ --requirement=/dev/stdin \
--dest=$@ \
--exists-action=i
@@ -732,6 +748,21 @@ requirements/vendor:
--dest=$@ \
--exists-action=i
+requirements/requirements_local.txt:
+ @echo "This is going to take a while..."
+ pip download \
+ --requirement=requirements/requirements_git.txt \
+ --no-deps \
+ --exists-action=w \
+ --dest=requirements/vendor 2>/dev/null | sed -n 's/^\s*Saved\s*//p' > $@
+
+requirements/requirements_ansible_local.txt:
+ pip download \
+ --requirement=requirements/requirements_ansible_git.txt \
+ --no-deps \
+ --exists-action=w \
+ --dest=requirements/vendor 2>/dev/null | sed -n 's/^\s*Saved\s*//p' > $@
+
rpm-build/$(RPM_NVR).src.rpm: /etc/mock/$(MOCK_CFG).cfg
$(MOCK_BIN) -r $(MOCK_CFG) --resultdir rpm-build --buildsrpm --spec rpm-build/$(NAME).spec --sources rpm-build \
--define "tower_version $(VERSION)" --define "tower_release $(RELEASE)" $(SCL_DEFINES)
diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py
index d0c53f4fc0..f84ab0df4f 100644
--- a/awx/lib/tests/test_display_callback.py
+++ b/awx/lib/tests/test_display_callback.py
@@ -60,7 +60,7 @@ def executor(tmpdir_factory, request):
cli = PlaybookCLI(['', 'playbook.yml'])
cli.parse()
- options = cli.parser.parse_args([])[0]
+ options = cli.parser.parse_args(['-v'])[0]
loader = DataLoader()
variable_manager = VariableManager()
inventory = Inventory(loader=loader, variable_manager=variable_manager,
diff --git a/awx/lib/tower_display_callback/module.py b/awx/lib/tower_display_callback/module.py
index 59575d7989..76dcb5be7f 100644
--- a/awx/lib/tower_display_callback/module.py
+++ b/awx/lib/tower_display_callback/module.py
@@ -30,6 +30,8 @@ from ansible.plugins.callback.default import CallbackModule as DefaultCallbackMo
from .events import event_context
from .minimal import CallbackModule as MinimalCallbackModule
+CENSORED = "the output has been hidden due to the fact that 'no_log: true' was specified for this result" # noqa
+
class BaseCallbackModule(CallbackBase):
'''
@@ -69,8 +71,12 @@ class BaseCallbackModule(CallbackBase):
else:
task = None
- if event_data.get('res') and event_data['res'].get('_ansible_no_log', False):
- event_data['res'] = {'censored': "the output has been hidden due to the fact that 'no_log: true' was specified for this result"} # noqa
+ if event_data.get('res'):
+ if event_data['res'].get('_ansible_no_log', False):
+ event_data['res'] = {'censored': CENSORED}
+ for i, item in enumerate(event_data['res'].get('results', [])):
+ if event_data['res']['results'][i].get('_ansible_no_log', False):
+ event_data['res']['results'][i] = {'censored': CENSORED}
with event_context.display_lock:
try:
diff --git a/awx/main/conf.py b/awx/main/conf.py
index 098b84f639..674c7fabee 100644
--- a/awx/main/conf.py
+++ b/awx/main/conf.py
@@ -242,9 +242,11 @@ register(
field_class=fields.IntegerField,
allow_null=True,
label=_('Logging Aggregator Port'),
- help_text=_('Port on Logging Aggregator to send logs to (if required).'),
+ help_text=_('Port on Logging Aggregator to send logs to (if required and not'
+ ' provided in Logging Aggregator).'),
category=_('Logging'),
category_slug='logging',
+ required=False
)
register(
'LOG_AGGREGATOR_TYPE',
diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py
index 3ae26eaf71..e818b5b648 100644
--- a/awx/main/models/mixins.py
+++ b/awx/main/models/mixins.py
@@ -130,13 +130,18 @@ class SurveyJobTemplateMixin(models.Model):
for survey_element in self.survey_spec.get("spec", []):
default = survey_element.get('default')
variable_key = survey_element.get('variable')
+
if survey_element.get('type') == 'password':
if variable_key in kwargs_extra_vars and default:
kw_value = kwargs_extra_vars[variable_key]
if kw_value.startswith('$encrypted$') and kw_value != default:
kwargs_extra_vars[variable_key] = default
+
if default is not None:
- extra_vars[variable_key] = default
+ data = {variable_key: default}
+ errors = self._survey_element_validation(survey_element, data)
+ if not errors:
+ extra_vars[variable_key] = default
# Overwrite job template extra vars with explicit job extra vars
# and add on job extra vars
@@ -144,6 +149,65 @@ class SurveyJobTemplateMixin(models.Model):
kwargs['extra_vars'] = json.dumps(extra_vars)
return kwargs
+ def _survey_element_validation(self, survey_element, data):
+ errors = []
+ if survey_element['variable'] not in data and survey_element['required']:
+ errors.append("'%s' value missing" % survey_element['variable'])
+ elif survey_element['type'] in ["textarea", "text", "password"]:
+ if survey_element['variable'] in data:
+ if type(data[survey_element['variable']]) not in (str, unicode):
+ errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']],
+ survey_element['variable']))
+ return errors
+ if 'min' in survey_element and survey_element['min'] not in ["", None] and len(data[survey_element['variable']]) < int(survey_element['min']):
+ errors.append("'%s' value %s is too small (length is %s must be at least %s)." %
+ (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min']))
+ if 'max' in survey_element and survey_element['max'] not in ["", None] and len(data[survey_element['variable']]) > int(survey_element['max']):
+ errors.append("'%s' value %s is too large (must be no more than %s)." %
+ (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
+ elif survey_element['type'] == 'integer':
+ if survey_element['variable'] in data:
+ if type(data[survey_element['variable']]) != int:
+ errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']],
+ survey_element['variable']))
+ return errors
+ if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \
+ data[survey_element['variable']] < int(survey_element['min']):
+ errors.append("'%s' value %s is too small (must be at least %s)." %
+ (survey_element['variable'], data[survey_element['variable']], survey_element['min']))
+ if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \
+ data[survey_element['variable']] > int(survey_element['max']):
+ errors.append("'%s' value %s is too large (must be no more than %s)." %
+ (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
+ elif survey_element['type'] == 'float':
+ if survey_element['variable'] in data:
+ if type(data[survey_element['variable']]) not in (float, int):
+ errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']],
+ survey_element['variable']))
+ return errors
+ if 'min' in survey_element and survey_element['min'] not in ["", None] and data[survey_element['variable']] < float(survey_element['min']):
+ errors.append("'%s' value %s is too small (must be at least %s)." %
+ (survey_element['variable'], data[survey_element['variable']], survey_element['min']))
+ if 'max' in survey_element and survey_element['max'] not in ["", None] and data[survey_element['variable']] > float(survey_element['max']):
+ errors.append("'%s' value %s is too large (must be no more than %s)." %
+ (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
+ elif survey_element['type'] == 'multiselect':
+ if survey_element['variable'] in data:
+ if type(data[survey_element['variable']]) != list:
+ errors.append("'%s' value is expected to be a list." % survey_element['variable'])
+ else:
+ for val in data[survey_element['variable']]:
+ if val not in survey_element['choices']:
+ errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'],
+ survey_element['choices']))
+ elif survey_element['type'] == 'multiplechoice':
+ if survey_element['variable'] in data:
+ if data[survey_element['variable']] not in survey_element['choices']:
+ errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']],
+ survey_element['variable'],
+ survey_element['choices']))
+ return errors
+
def survey_variable_validation(self, data):
errors = []
if not self.survey_enabled:
@@ -153,62 +217,7 @@ class SurveyJobTemplateMixin(models.Model):
if 'description' not in self.survey_spec:
errors.append("'description' missing from survey spec.")
for survey_element in self.survey_spec.get("spec", []):
- if survey_element['variable'] not in data and \
- survey_element['required']:
- errors.append("'%s' value missing" % survey_element['variable'])
- elif survey_element['type'] in ["textarea", "text", "password"]:
- if survey_element['variable'] in data:
- if type(data[survey_element['variable']]) not in (str, unicode):
- errors.append("Value %s for '%s' expected to be a string." % (data[survey_element['variable']],
- survey_element['variable']))
- continue
- if 'min' in survey_element and survey_element['min'] not in ["", None] and len(data[survey_element['variable']]) < int(survey_element['min']):
- errors.append("'%s' value %s is too small (length is %s must be at least %s)." %
- (survey_element['variable'], data[survey_element['variable']], len(data[survey_element['variable']]), survey_element['min']))
- if 'max' in survey_element and survey_element['max'] not in ["", None] and len(data[survey_element['variable']]) > int(survey_element['max']):
- errors.append("'%s' value %s is too large (must be no more than %s)." %
- (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
- elif survey_element['type'] == 'integer':
- if survey_element['variable'] in data:
- if type(data[survey_element['variable']]) != int:
- errors.append("Value %s for '%s' expected to be an integer." % (data[survey_element['variable']],
- survey_element['variable']))
- continue
- if 'min' in survey_element and survey_element['min'] not in ["", None] and survey_element['variable'] in data and \
- data[survey_element['variable']] < int(survey_element['min']):
- errors.append("'%s' value %s is too small (must be at least %s)." %
- (survey_element['variable'], data[survey_element['variable']], survey_element['min']))
- if 'max' in survey_element and survey_element['max'] not in ["", None] and survey_element['variable'] in data and \
- data[survey_element['variable']] > int(survey_element['max']):
- errors.append("'%s' value %s is too large (must be no more than %s)." %
- (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
- elif survey_element['type'] == 'float':
- if survey_element['variable'] in data:
- if type(data[survey_element['variable']]) not in (float, int):
- errors.append("Value %s for '%s' expected to be a numeric type." % (data[survey_element['variable']],
- survey_element['variable']))
- continue
- if 'min' in survey_element and survey_element['min'] not in ["", None] and data[survey_element['variable']] < float(survey_element['min']):
- errors.append("'%s' value %s is too small (must be at least %s)." %
- (survey_element['variable'], data[survey_element['variable']], survey_element['min']))
- if 'max' in survey_element and survey_element['max'] not in ["", None] and data[survey_element['variable']] > float(survey_element['max']):
- errors.append("'%s' value %s is too large (must be no more than %s)." %
- (survey_element['variable'], data[survey_element['variable']], survey_element['max']))
- elif survey_element['type'] == 'multiselect':
- if survey_element['variable'] in data:
- if type(data[survey_element['variable']]) != list:
- errors.append("'%s' value is expected to be a list." % survey_element['variable'])
- else:
- for val in data[survey_element['variable']]:
- if val not in survey_element['choices']:
- errors.append("Value %s for '%s' expected to be one of %s." % (val, survey_element['variable'],
- survey_element['choices']))
- elif survey_element['type'] == 'multiplechoice':
- if survey_element['variable'] in data:
- if data[survey_element['variable']] not in survey_element['choices']:
- errors.append("Value %s for '%s' expected to be one of %s." % (data[survey_element['variable']],
- survey_element['variable'],
- survey_element['choices']))
+ errors += self._survey_element_validation(survey_element, data)
return errors
diff --git a/awx/main/tests/unit/models/test_survey_models.py b/awx/main/tests/unit/models/test_survey_models.py
index 584a4cc7f0..eefc5d97ab 100644
--- a/awx/main/tests/unit/models/test_survey_models.py
+++ b/awx/main/tests/unit/models/test_survey_models.py
@@ -4,6 +4,7 @@ import json
from awx.main.tasks import RunJob
from awx.main.models import (
Job,
+ JobTemplate,
WorkflowJobTemplate
)
@@ -78,6 +79,18 @@ def test_job_args_unredacted_passwords(job):
assert extra_vars['secret_key'] == 'my_password'
+def test_update_kwargs_survey_invalid_default(survey_spec_factory):
+ spec = survey_spec_factory('var2')
+ spec['spec'][0]['required'] = False
+ spec['spec'][0]['min'] = 3
+ spec['spec'][0]['default'] = 1
+ jt = JobTemplate(name="test-jt", survey_spec=spec, survey_enabled=True, extra_vars="var2: 2")
+ defaulted_extra_vars = jt._update_unified_job_kwargs()
+ assert 'extra_vars' in defaulted_extra_vars
+ # Make sure we did not set the invalid default of 1
+ assert json.loads(defaulted_extra_vars['extra_vars'])['var2'] == 2
+
+
class TestWorkflowSurveys:
def test_update_kwargs_survey_defaults(self, survey_spec_factory):
"Assure that the survey default over-rides a JT variable"
diff --git a/awx/main/tests/unit/utils/test_handlers.py b/awx/main/tests/unit/utils/test_handlers.py
index ce94ce0907..e398e72ff8 100644
--- a/awx/main/tests/unit/utils/test_handlers.py
+++ b/awx/main/tests/unit/utils/test_handlers.py
@@ -147,7 +147,17 @@ def test_https_logging_handler_splunk_auth_info():
('http://localhost', None, 'http://localhost'),
('http://localhost', 80, 'http://localhost'),
('http://localhost', 8080, 'http://localhost:8080'),
- ('https://localhost', 443, 'https://localhost:443')
+ ('https://localhost', 443, 'https://localhost:443'),
+ ('ftp://localhost', 443, 'ftp://localhost:443'),
+ ('https://localhost:550', 443, 'https://localhost:550'),
+ ('https://localhost:yoho/foobar', 443, 'https://localhost:443/foobar'),
+ ('https://localhost:yoho/foobar', None, 'https://localhost:yoho/foobar'),
+ ('http://splunk.server:8088/services/collector/event', 80,
+ 'http://splunk.server:8088/services/collector/event'),
+ ('http://splunk.server/services/collector/event', 80,
+ 'http://splunk.server/services/collector/event'),
+ ('http://splunk.server/services/collector/event', 8088,
+ 'http://splunk.server:8088/services/collector/event'),
])
def test_https_logging_handler_http_host_format(host, port, normalized):
handler = HTTPSHandler(host=host, port=port)
diff --git a/awx/main/utils/handlers.py b/awx/main/utils/handlers.py
index c8986ea596..2b80a492ed 100644
--- a/awx/main/utils/handlers.py
+++ b/awx/main/utils/handlers.py
@@ -6,6 +6,7 @@ import logging
import json
import requests
import time
+import urlparse
from concurrent.futures import ThreadPoolExecutor
from copy import copy
from requests.exceptions import RequestException
@@ -148,10 +149,21 @@ class BaseHTTPSHandler(logging.Handler):
def get_http_host(self):
host = self.host or ''
- if not host.startswith('http'):
- host = 'http://%s' % self.host
- if self.port != 80 and self.port is not None:
- host = '%s:%s' % (host, str(self.port))
+ # urlparse requires scheme to be provided, default to use http if
+ # missing
+ if not urlparse.urlsplit(host).scheme:
+ host = 'http://%s' % host
+ parsed = urlparse.urlsplit(host)
+ # Insert self.port if its special and port number is either not
+ # given in host or given as non-numerical
+ try:
+ port = parsed.port or self.port
+ except ValueError:
+ port = self.port
+ if port not in (80, None):
+ new_netloc = '%s:%s' % (parsed.hostname, port)
+ return urlparse.urlunsplit((parsed.scheme, new_netloc, parsed.path,
+ parsed.query, parsed.fragment))
return host
def get_post_kwargs(self, payload_input):
diff --git a/awx/playbooks/project_update.yml b/awx/playbooks/project_update.yml
index c3803c7118..bfabd47a98 100644
--- a/awx/playbooks/project_update.yml
+++ b/awx/playbooks/project_update.yml
@@ -108,8 +108,8 @@
- name: update project using insights
uri:
url: "{{insights_url}}/r/insights/v1/maintenance?ansible=true"
- user: "{{scm_username|quote}}"
- password: "{{scm_password|quote}}"
+ user: "{{scm_username}}"
+ password: "{{scm_password}}"
force_basic_auth: yes
when: scm_type == 'insights'
register: insights_output
@@ -124,8 +124,8 @@
get_url:
url: "{{insights_url}}/r/insights/v3/maintenance/{{item.maintenance_id}}/playbook"
dest: "{{project_path|quote}}/{{item.name}}-{{item.maintenance_id}}.yml"
- url_username: "{{scm_username|quote}}"
- url_password: "{{scm_password|quote}}"
+ url_username: "{{scm_username}}"
+ url_password: "{{scm_password}}"
force_basic_auth: yes
force: yes
when: scm_type == 'insights' and item.name != None
@@ -136,8 +136,8 @@
get_url:
url: "{{insights_url}}/r/insights/v3/maintenance/{{item.maintenance_id}}/playbook"
dest: "{{project_path|quote}}/insights-plan-{{item.maintenance_id}}.yml"
- url_username: "{{scm_username|quote}}"
- url_password: "{{scm_password|quote}}"
+ url_username: "{{scm_username}}"
+ url_password: "{{scm_password}}"
force_basic_auth: yes
force: yes
when: scm_type == 'insights' and item.name == None
diff --git a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html
index 768043addc..7da83dfb43 100644
--- a/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html
+++ b/awx/ui/client/src/job-results/host-event/host-event-modal.partial.html
@@ -9,7 +9,7 @@
{{event.host_name}}
-