diff --git a/awx/main/tasks.py b/awx/main/tasks.py index f801e3be58..a7d4e421e9 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -28,7 +28,7 @@ from django.utils.timezone import now # AWX from awx.main.models import Job, ProjectUpdate -from awx.main.utils import get_ansible_version, decrypt_field +from awx.main.utils import get_ansible_version, decrypt_field, update_scm_url __all__ = ['RunJob', 'RunProjectUpdate'] @@ -232,6 +232,7 @@ class BaseTask(Task): return instance = self.update_model(pk, status='running') status, stdout, tb = 'error', '', '' + output_replacements = [] try: kwargs['ssh_key_path'] = self.build_ssh_key_path(instance, **kwargs) kwargs['passwords'] = self.build_passwords(instance, **kwargs) @@ -452,45 +453,37 @@ class RunProjectUpdate(BaseTask): Build environment dictionary for ansible-playbook. ''' env = super(RunProjectUpdate, self).build_env(project_update, **kwargs) + env['ANSIBLE_ASK_PASS'] = str(False) env['ANSIBLE_ASK_SUDO_PASS'] = str(False) env['DISPLAY'] = '' # Prevent stupid password popup when running tests. return env - def update_url_auth(self, url, username=None, password=None): - parts = urlparse.urlsplit(url) - netloc_username = username or parts.username or '' - netloc_password = password or parts.password or '' - if netloc_username: - netloc = u':'.join(filter(None, [netloc_username, netloc_password])) - else: - netlock = u'' - netloc = u'@'.join(filter(None, [netloc, parts.hostname])) - netloc = u':'.join(filter(None, [netloc, parts.port])) - return urlparse.urlunsplit([parts.scheme, netloc, parts.path, - parts.query, parts.fragment]) - def _build_scm_url_extra_vars(self, project_update, **kwargs): ''' Helper method to build SCM url and extra vars with parameters needed for authentication. ''' + # FIXME: May need to pull username/password out of URL in other cases. extra_vars = {} project = project_update.project + scm_type = project.scm_type scm_url = project.scm_url scm_username = kwargs.get('passwords', {}).get('scm_username', '') scm_password = kwargs.get('passwords', {}).get('scm_password', '') if scm_username and scm_password not in ('ASK', ''): - if project.scm_type == 'svn': + if scm_type == 'svn': extra_vars['scm_username'] = scm_username extra_vars['scm_password'] = scm_password else: - scm_url = self.update_url_auth(scm_url, scm_username, - scm_password) + scm_url = update_scm_url(scm_type, scm_url, scm_username, + scm_password) elif scm_username: - if project.scm_type == 'svn': + if scm_type == 'svn': extra_vars['scm_username'] = scm_username else: - scm_url = self.update_url_auth(scm_url, scm_username) + scm_url = update_scm_url(scm_type, scm_url, scm_username) + else: + scm_url = update_scm_url(scm_type, scm_url) return scm_url, extra_vars def build_args(self, project_update, **kwargs): diff --git a/awx/main/tests/projects.py b/awx/main/tests/projects.py index 044b47b939..85d89ef0ab 100644 --- a/awx/main/tests/projects.py +++ b/awx/main/tests/projects.py @@ -5,6 +5,8 @@ import datetime import json import os +import re +import subprocess import tempfile import urlparse @@ -21,7 +23,7 @@ from django.utils.timezone import now from awx.main.models import * from awx.main.tests.base import BaseTest, BaseTransactionTest from awx.main.tests.tasks import TEST_SSH_KEY_DATA_LOCKED, TEST_SSH_KEY_DATA_UNLOCK -from awx.main.utils import decrypt_field +from awx.main.utils import decrypt_field, update_scm_url TEST_PLAYBOOK = '''- hosts: mygroup gather_facts: false @@ -631,6 +633,175 @@ class ProjectUpdatesTest(BaseTransactionTest): self._temp_project_dirs.append(project_path) return project + def test_update_scm_url(self): + # Handle all of the URL formats supported by the SCM systems: + urls_to_test = [ + # (scm type, original url, new url, new url with username, new url with username and password) + + # git: https://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + # - ssh://[user@]host.xz[:port]/path/to/repo.git/ + ('git', 'ssh://host.xz/path/to/repo.git/', None, 'ssh://testuser@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ssh://host.xz:1022/path/to/repo.git', None, 'ssh://testuser@host.xz:1022/path/to/repo.git', 'ssh://testuser:testpass@host.xz:1022/path/to/repo.git'), + ('git', 'ssh://user@host.xz/path/to/repo.git/', None, 'ssh://testuser@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ssh://user@host.xz:1022/path/to/repo.git', None, 'ssh://testuser@host.xz:1022/path/to/repo.git', 'ssh://testuser:testpass@host.xz:1022/path/to/repo.git'), + ('git', 'ssh://user:pass@host.xz/path/to/repo.git/', None, 'ssh://testuser:pass@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ssh://user:pass@host.xz:1022/path/to/repo.git', None, 'ssh://testuser:pass@host.xz:1022/path/to/repo.git', 'ssh://testuser:testpass@host.xz:1022/path/to/repo.git'), + # - git://host.xz[:port]/path/to/repo.git/ (doesn't really support authentication) + ('git', 'git://host.xz/path/to/repo.git/', None, 'git://testuser@host.xz/path/to/repo.git/', 'git://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'git://host.xz:9418/path/to/repo.git', None, 'git://testuser@host.xz:9418/path/to/repo.git', 'git://testuser:testpass@host.xz:9418/path/to/repo.git'), + ('git', 'git://user@host.xz/path/to/repo.git/', None, 'git://testuser@host.xz/path/to/repo.git/', 'git://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'git://user@host.xz:9418/path/to/repo.git', None, 'git://testuser@host.xz:9418/path/to/repo.git', 'git://testuser:testpass@host.xz:9418/path/to/repo.git'), + # - http[s]://host.xz[:port]/path/to/repo.git/ + ('git', 'http://host.xz/path/to/repo.git/', None, 'http://testuser@host.xz/path/to/repo.git/', 'http://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'http://host.xz:8080/path/to/repo.git', None, 'http://testuser@host.xz:8080/path/to/repo.git', 'http://testuser:testpass@host.xz:8080/path/to/repo.git'), + ('git', 'http://user@host.xz/path/to/repo.git/', None, 'http://testuser@host.xz/path/to/repo.git/', 'http://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'http://user@host.xz:8080/path/to/repo.git', None, 'http://testuser@host.xz:8080/path/to/repo.git', 'http://testuser:testpass@host.xz:8080/path/to/repo.git'), + ('git', 'http://user:pass@host.xz/path/to/repo.git/', None, 'http://testuser:pass@host.xz/path/to/repo.git/', 'http://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'http://user:pass@host.xz:8080/path/to/repo.git', None, 'http://testuser:pass@host.xz:8080/path/to/repo.git', 'http://testuser:testpass@host.xz:8080/path/to/repo.git'), + ('git', 'https://host.xz/path/to/repo.git/', None, 'https://testuser@host.xz/path/to/repo.git/', 'https://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'https://host.xz:8443/path/to/repo.git', None, 'https://testuser@host.xz:8443/path/to/repo.git', 'https://testuser:testpass@host.xz:8443/path/to/repo.git'), + ('git', 'https://user@host.xz/path/to/repo.git/', None, 'https://testuser@host.xz/path/to/repo.git/', 'https://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'https://user@host.xz:8443/path/to/repo.git', None, 'https://testuser@host.xz:8443/path/to/repo.git', 'https://testuser:testpass@host.xz:8443/path/to/repo.git'), + ('git', 'https://user:pass@host.xz/path/to/repo.git/', None, 'https://testuser:pass@host.xz/path/to/repo.git/', 'https://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'https://user:pass@host.xz:8443/path/to/repo.git', None, 'https://testuser:pass@host.xz:8443/path/to/repo.git', 'https://testuser:testpass@host.xz:8443/path/to/repo.git'), + # - ftp[s]://host.xz[:port]/path/to/repo.git/ + ('git', 'ftp://host.xz/path/to/repo.git/', None, 'ftp://testuser@host.xz/path/to/repo.git/', 'ftp://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftp://host.xz:8021/path/to/repo.git', None, 'ftp://testuser@host.xz:8021/path/to/repo.git', 'ftp://testuser:testpass@host.xz:8021/path/to/repo.git'), + ('git', 'ftp://user@host.xz/path/to/repo.git/', None, 'ftp://testuser@host.xz/path/to/repo.git/', 'ftp://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftp://user@host.xz:8021/path/to/repo.git', None, 'ftp://testuser@host.xz:8021/path/to/repo.git', 'ftp://testuser:testpass@host.xz:8021/path/to/repo.git'), + ('git', 'ftp://user:pass@host.xz/path/to/repo.git/', None, 'ftp://testuser:pass@host.xz/path/to/repo.git/', 'ftp://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftp://user:pass@host.xz:8021/path/to/repo.git', None, 'ftp://testuser:pass@host.xz:8021/path/to/repo.git', 'ftp://testuser:testpass@host.xz:8021/path/to/repo.git'), + ('git', 'ftps://host.xz/path/to/repo.git/', None, 'ftps://testuser@host.xz/path/to/repo.git/', 'ftps://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftps://host.xz:8990/path/to/repo.git', None, 'ftps://testuser@host.xz:8990/path/to/repo.git', 'ftps://testuser:testpass@host.xz:8990/path/to/repo.git'), + ('git', 'ftps://user@host.xz/path/to/repo.git/', None, 'ftps://testuser@host.xz/path/to/repo.git/', 'ftps://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftps://user@host.xz:8990/path/to/repo.git', None, 'ftps://testuser@host.xz:8990/path/to/repo.git', 'ftps://testuser:testpass@host.xz:8990/path/to/repo.git'), + ('git', 'ftps://user:pass@host.xz/path/to/repo.git/', None, 'ftps://testuser:pass@host.xz/path/to/repo.git/', 'ftps://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'ftps://user:pass@host.xz:8990/path/to/repo.git', None, 'ftps://testuser:pass@host.xz:8990/path/to/repo.git', 'ftps://testuser:testpass@host.xz:8990/path/to/repo.git'), + # - rsync://host.xz/path/to/repo.git/ + ('git', 'rsync://host.xz/path/to/repo.git/', None, 'rsync://testuser@host.xz/path/to/repo.git/', 'rsync://testuser:testpass@host.xz/path/to/repo.git/'), + # - [user@]host.xz:path/to/repo.git/ (SCP style) + ('git', 'host.xz:path/to/repo.git/', 'ssh://host.xz/path/to/repo.git/', 'ssh://testuser@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'user@host.xz:path/to/repo.git/', 'ssh://user@host.xz/path/to/repo.git/', 'ssh://testuser@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + ('git', 'user:pass@host.xz:path/to/repo.git/', 'ssh://user:pass@host.xz/path/to/repo.git/', 'ssh://testuser:pass@host.xz/path/to/repo.git/', 'ssh://testuser:testpass@host.xz/path/to/repo.git/'), + # - /path/to/repo.git/ (local file) + ('git', '/path/to/repo.git', 'file:///path/to/repo.git', 'file:///path/to/repo.git', 'file:///path/to/repo.git'), + ('git', 'path/to/repo.git', 'file:///path/to/repo.git', 'file:///path/to/repo.git', 'file:///path/to/repo.git'), + # - file:///path/to/repo.git/ + ('git', 'file:///path/to/repo.git', None, None, None), + ('git', 'file://localhost/path/to/repo.git', None, None, None), + + # hg: http://www.selenic.com/mercurial/hg.1.html#url-paths + # - local/filesystem/path[#revision] + ('hg', '/path/to/repo', 'file:///path/to/repo', 'file:///path/to/repo', 'file:///path/to/repo'), + ('hg', 'path/to/repo/', 'file:///path/to/repo/', 'file:///path/to/repo/', 'file:///path/to/repo/'), + ('hg', '/path/to/repo#rev', 'file:///path/to/repo#rev', 'file:///path/to/repo#rev', 'file:///path/to/repo#rev'), + ('hg', 'path/to/repo/#rev', 'file:///path/to/repo/#rev', 'file:///path/to/repo/#rev', 'file:///path/to/repo/#rev'), + # - file://local/filesystem/path[#revision] + ('hg', 'file:///path/to/repo', None, None, None), + ('hg', 'file://localhost/path/to/repo/', None, None, None), + ('hg', 'file:///path/to/repo#rev', None, None, None), + ('hg', 'file://localhost/path/to/repo/#rev', None, None, None), + # - http://[user[:pass]@]host[:port]/[path][#revision] + ('hg', 'http://host.xz/path/to/repo/', None, 'http://testuser@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'http://host.xz:8080/path/to/repo', None, 'http://testuser@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + ('hg', 'http://user@host.xz/path/to/repo/', None, 'http://testuser@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'http://user@host.xz:8080/path/to/repo', None, 'http://testuser@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + ('hg', 'http://user:pass@host.xz/path/to/repo/', None, 'http://testuser:pass@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'http://user:pass@host.xz:8080/path/to/repo', None, 'http://testuser:pass@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + ('hg', 'http://host.xz/path/to/repo/#rev', None, 'http://testuser@host.xz/path/to/repo/#rev', 'http://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'http://host.xz:8080/path/to/repo#rev', None, 'http://testuser@host.xz:8080/path/to/repo#rev', 'http://testuser:testpass@host.xz:8080/path/to/repo#rev'), + ('hg', 'http://user@host.xz/path/to/repo/#rev', None, 'http://testuser@host.xz/path/to/repo/#rev', 'http://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'http://user@host.xz:8080/path/to/repo#rev', None, 'http://testuser@host.xz:8080/path/to/repo#rev', 'http://testuser:testpass@host.xz:8080/path/to/repo#rev'), + ('hg', 'http://user:pass@host.xz/path/to/repo/#rev', None, 'http://testuser:pass@host.xz/path/to/repo/#rev', 'http://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'http://user:pass@host.xz:8080/path/to/repo#rev', None, 'http://testuser:pass@host.xz:8080/path/to/repo#rev', 'http://testuser:testpass@host.xz:8080/path/to/repo#rev'), + # - https://[user[:pass]@]host[:port]/[path][#revision] + ('hg', 'https://host.xz/path/to/repo/', None, 'https://testuser@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'https://host.xz:8443/path/to/repo', None, 'https://testuser@host.xz:8443/path/to/repo', 'https://testuser:testpass@host.xz:8443/path/to/repo'), + ('hg', 'https://user@host.xz/path/to/repo/', None, 'https://testuser@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'https://user@host.xz:8443/path/to/repo', None, 'https://testuser@host.xz:8443/path/to/repo', 'https://testuser:testpass@host.xz:8443/path/to/repo'), + ('hg', 'https://user:pass@host.xz/path/to/repo/', None, 'https://testuser:pass@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'https://user:pass@host.xz:8443/path/to/repo', None, 'https://testuser:pass@host.xz:8443/path/to/repo', 'https://testuser:testpass@host.xz:8443/path/to/repo'), + ('hg', 'https://host.xz/path/to/repo/#rev', None, 'https://testuser@host.xz/path/to/repo/#rev', 'https://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'https://host.xz:8443/path/to/repo#rev', None, 'https://testuser@host.xz:8443/path/to/repo#rev', 'https://testuser:testpass@host.xz:8443/path/to/repo#rev'), + ('hg', 'https://user@host.xz/path/to/repo/#rev', None, 'https://testuser@host.xz/path/to/repo/#rev', 'https://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'https://user@host.xz:8443/path/to/repo#rev', None, 'https://testuser@host.xz:8443/path/to/repo#rev', 'https://testuser:testpass@host.xz:8443/path/to/repo#rev'), + ('hg', 'https://user:pass@host.xz/path/to/repo/#rev', None, 'https://testuser:pass@host.xz/path/to/repo/#rev', 'https://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'https://user:pass@host.xz:8443/path/to/repo#rev', None, 'https://testuser:pass@host.xz:8443/path/to/repo#rev', 'https://testuser:testpass@host.xz:8443/path/to/repo#rev'), + # - ssh://[user@]host[:port]/[path][#revision] + ('hg', 'ssh://host.xz/path/to/repo/', None, 'ssh://testuser@host.xz/path/to/repo/', 'ssh://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'ssh://host.xz:1022/path/to/repo', None, 'ssh://testuser@host.xz:1022/path/to/repo', 'ssh://testuser:testpass@host.xz:1022/path/to/repo'), + ('hg', 'ssh://user@host.xz/path/to/repo/', None, 'ssh://testuser@host.xz/path/to/repo/', 'ssh://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'ssh://user@host.xz:1022/path/to/repo', None, 'ssh://testuser@host.xz:1022/path/to/repo', 'ssh://testuser:testpass@host.xz:1022/path/to/repo'), + ('hg', 'ssh://user:pass@host.xz/path/to/repo/', None, 'ssh://testuser:pass@host.xz/path/to/repo/', 'ssh://testuser:testpass@host.xz/path/to/repo/'), + ('hg', 'ssh://user:pass@host.xz:1022/path/to/repo', None, 'ssh://testuser:pass@host.xz:1022/path/to/repo', 'ssh://testuser:testpass@host.xz:1022/path/to/repo'), + ('hg', 'ssh://host.xz/path/to/repo/#rev', None, 'ssh://testuser@host.xz/path/to/repo/#rev', 'ssh://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'ssh://host.xz:1022/path/to/repo#rev', None, 'ssh://testuser@host.xz:1022/path/to/repo#rev', 'ssh://testuser:testpass@host.xz:1022/path/to/repo#rev'), + ('hg', 'ssh://user@host.xz/path/to/repo/#rev', None, 'ssh://testuser@host.xz/path/to/repo/#rev', 'ssh://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'ssh://user@host.xz:1022/path/to/repo#rev', None, 'ssh://testuser@host.xz:1022/path/to/repo#rev', 'ssh://testuser:testpass@host.xz:1022/path/to/repo#rev'), + ('hg', 'ssh://user:pass@host.xz/path/to/repo/#rev', None, 'ssh://testuser:pass@host.xz/path/to/repo/#rev', 'ssh://testuser:testpass@host.xz/path/to/repo/#rev'), + ('hg', 'ssh://user:pass@host.xz:1022/path/to/repo#rev', None, 'ssh://testuser:pass@host.xz:1022/path/to/repo#rev', 'ssh://testuser:testpass@host.xz:1022/path/to/repo#rev'), + + # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls + # - file:/// Direct repository access (on local disk) + ('svn', 'file:///path/to/repo', None, None, None), + ('svn', 'file://localhost/path/to/repo/', None, None, None), + # - http:// Access via WebDAV protocol to Subversion-aware Apache server + ('svn', 'http://host.xz/path/to/repo/', None, 'http://testuser@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'http://host.xz:8080/path/to/repo', None, 'http://testuser@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + ('svn', 'http://user@host.xz/path/to/repo/', None, 'http://testuser@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'http://user@host.xz:8080/path/to/repo', None, 'http://testuser@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + ('svn', 'http://user:pass@host.xz/path/to/repo/', None, 'http://testuser:pass@host.xz/path/to/repo/', 'http://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'http://user:pass@host.xz:8080/path/to/repo', None, 'http://testuser:pass@host.xz:8080/path/to/repo', 'http://testuser:testpass@host.xz:8080/path/to/repo'), + # - https:// Same as http://, but with SSL encryption + ('svn', 'https://host.xz/path/to/repo/', None, 'https://testuser@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'https://host.xz:8080/path/to/repo', None, 'https://testuser@host.xz:8080/path/to/repo', 'https://testuser:testpass@host.xz:8080/path/to/repo'), + ('svn', 'https://user@host.xz/path/to/repo/', None, 'https://testuser@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'https://user@host.xz:8080/path/to/repo', None, 'https://testuser@host.xz:8080/path/to/repo', 'https://testuser:testpass@host.xz:8080/path/to/repo'), + ('svn', 'https://user:pass@host.xz/path/to/repo/', None, 'https://testuser:pass@host.xz/path/to/repo/', 'https://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'https://user:pass@host.xz:8080/path/to/repo', None, 'https://testuser:pass@host.xz:8080/path/to/repo', 'https://testuser:testpass@host.xz:8080/path/to/repo'), + # - svn:// Access via custom protocol to an svnserve server + ('svn', 'svn://host.xz/path/to/repo/', None, 'svn://testuser@host.xz/path/to/repo/', 'svn://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn://host.xz:3690/path/to/repo', None, 'svn://testuser@host.xz:3690/path/to/repo', 'svn://testuser:testpass@host.xz:3690/path/to/repo'), + ('svn', 'svn://user@host.xz/path/to/repo/', None, 'svn://testuser@host.xz/path/to/repo/', 'svn://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn://user@host.xz:3690/path/to/repo', None, 'svn://testuser@host.xz:3690/path/to/repo', 'svn://testuser:testpass@host.xz:3690/path/to/repo'), + ('svn', 'svn://user:pass@host.xz/path/to/repo/', None, 'svn://testuser:pass@host.xz/path/to/repo/', 'svn://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn://user:pass@host.xz:3690/path/to/repo', None, 'svn://testuser:pass@host.xz:3690/path/to/repo', 'svn://testuser:testpass@host.xz:3690/path/to/repo'), + # - svn+ssh:// Same as svn://, but through an SSH tunnel + ('svn', 'svn+ssh://host.xz/path/to/repo/', None, 'svn+ssh://testuser@host.xz/path/to/repo/', 'svn+ssh://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn+ssh://host.xz:1022/path/to/repo', None, 'svn+ssh://testuser@host.xz:1022/path/to/repo', 'svn+ssh://testuser:testpass@host.xz:1022/path/to/repo'), + ('svn', 'svn+ssh://user@host.xz/path/to/repo/', None, 'svn+ssh://testuser@host.xz/path/to/repo/', 'svn+ssh://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn+ssh://user@host.xz:1022/path/to/repo', None, 'svn+ssh://testuser@host.xz:1022/path/to/repo', 'svn+ssh://testuser:testpass@host.xz:1022/path/to/repo'), + ('svn', 'svn+ssh://user:pass@host.xz/path/to/repo/', None, 'svn+ssh://testuser:pass@host.xz/path/to/repo/', 'svn+ssh://testuser:testpass@host.xz/path/to/repo/'), + ('svn', 'svn+ssh://user:pass@host.xz:1022/path/to/repo', None, 'svn+ssh://testuser:pass@host.xz:1022/path/to/repo', 'svn+ssh://testuser:testpass@host.xz:1022/path/to/repo'), + + # FIXME: Add some invalid URLs. + ] + for url_opts in urls_to_test: + scm_type, url, new_url, new_url_u, new_url_up = url_opts + new_url = new_url or url + new_url_u = new_url_u or url + new_url_up = new_url_up or url + if isinstance(new_url, Exception): + self.assertRaises(new_url, update_scm_url, scm_type, url) + else: + updated_url = update_scm_url(scm_type, url) + self.assertEqual(new_url, updated_url) + if isinstance(new_url_u, Exception): + self.assertRaises(new_url_u, update_scm_url, scm_type, + url, username='testuser') + else: + updated_url = update_scm_url(scm_type, url, + username='testuser') + self.assertEqual(new_url_u, updated_url) + if isinstance(new_url_up, Exception): + self.assertRaises(new_url_up, update_scm_url, scm_type, + url, username='testuser', password='testpass') + else: + updated_url = update_scm_url(scm_type, url, + username='testuser', + password='testpass') + self.assertEqual(new_url_up, updated_url) + def check_project_update(self, project, should_fail=False, **kwargs): pu = kwargs.pop('project_update', None) if not pu: @@ -638,9 +809,18 @@ class ProjectUpdatesTest(BaseTransactionTest): self.assertTrue(pu) pu = ProjectUpdate.objects.get(pk=pu.pk) if should_fail: - self.assertEqual(pu.status, 'failed', pu.result_stdout) + self.assertEqual(pu.status, 'failed', + pu.result_stdout + pu.result_traceback) else: - self.assertEqual(pu.status, 'successful', pu.result_stdout) + self.assertEqual(pu.status, 'successful', + pu.result_stdout + pu.result_traceback) + # Get the SCM URL from the job args, if it starts with a '/' we aren't + # handling the URL correctly. + scm_url_in_args_re = re.compile(r'\\(?:\\\\)??"scm_url\\(?:\\\\)??": \\(?:\\\\)??"(.*?)\\(?:\\\\)??"') + match = scm_url_in_args_re.search(pu.job_args) + self.assertTrue(match, pu.job_args) + scm_url_in_args = match.groups()[0] + self.assertFalse(scm_url_in_args.startswith('/'), scm_url_in_args) scm_password = kwargs.get('scm_password', decrypt_field(project, 'scm_password')) if scm_password not in ('', 'ASK'): @@ -791,6 +971,19 @@ class ProjectUpdatesTest(BaseTransactionTest): scm_url=scm_url, ) self.check_project_scm(project) + # Test passing username/password for public project. Though they're not + # needed, the update should still work. + scm_username = getattr(settings, 'TEST_GIT_USERNAME', '') + scm_password = getattr(settings, 'TEST_GIT_PASSWORD', '') + if scm_username or scm_password: + project2 = self.create_project( + name='my other public git project over https', + scm_type='git', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_update(project2) def test_private_git_project_over_https(self): scm_url = getattr(settings, 'TEST_GIT_PRIVATE_HTTPS', '') @@ -810,7 +1003,9 @@ class ProjectUpdatesTest(BaseTransactionTest): def test_private_git_project_over_ssh(self): scm_url = getattr(settings, 'TEST_GIT_PRIVATE_SSH', '') scm_key_data = getattr(settings, 'TEST_GIT_KEY_DATA', '') - if not all([scm_url, scm_key_data]): + scm_username = getattr(settings, 'TEST_GIT_USERNAME', '') + scm_password = 'blahblahblah'#getattr(settings, 'TEST_GIT_PASSWORD', '') + if not all([scm_url, scm_key_data, scm_username, scm_password]): self.skipTest('no private git repo defined for ssh!') project = self.create_project( name='my private git project over ssh', @@ -819,6 +1014,39 @@ class ProjectUpdatesTest(BaseTransactionTest): scm_key_data=scm_key_data, ) self.check_project_scm(project) + # Test project using SSH username/password instead of key. Should fail + # because of bad password, but never hang. + project2 = self.create_project( + name='my other private git project over ssh', + scm_type='git', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_update(project2, should_fail=True) + + def test_git_project_from_local_path(self): + # Create temp repository directory. + repo_dir = tempfile.mkdtemp() + self._temp_project_dirs.append(repo_dir) + handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir) + test_playbook_file = os.fdopen(handle, 'w') + test_playbook_file.write(TEST_PLAYBOOK) + test_playbook_file.close() + subprocess.check_call(['git', 'init', '.'], cwd=repo_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.check_call(['git', 'add', os.path.basename(playbook_path)], + cwd=repo_dir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.check_call(['git', 'commit', '-m', 'blah'], cwd=repo_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Now test project using local repo. + project = self.create_project( + name='my git project from local path', + scm_type='git', + scm_url=repo_dir, + ) + self.check_project_scm(project) def test_public_hg_project_over_https(self): scm_url = getattr(settings, 'TEST_HG_PUBLIC_HTTPS', @@ -831,6 +1059,19 @@ class ProjectUpdatesTest(BaseTransactionTest): scm_url=scm_url, ) self.check_project_scm(project) + # Test passing username/password for public project. Though they're not + # needed, the update should still work. + scm_username = getattr(settings, 'TEST_HG_USERNAME', '') + scm_password = getattr(settings, 'TEST_HG_PASSWORD', '') + if scm_username or scm_password: + project2 = self.create_project( + name='my other public hg project over https', + scm_type='hg', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_update(project2) def test_private_hg_project_over_https(self): scm_url = getattr(settings, 'TEST_HG_PRIVATE_HTTPS', '') @@ -850,7 +1091,9 @@ class ProjectUpdatesTest(BaseTransactionTest): def test_private_hg_project_over_ssh(self): scm_url = getattr(settings, 'TEST_HG_PRIVATE_SSH', '') scm_key_data = getattr(settings, 'TEST_HG_KEY_DATA', '') - if not all([scm_url, scm_key_data]): + scm_username = getattr(settings, 'TEST_HG_USERNAME', '') + scm_password = 'blahblahblah' + if not all([scm_url, scm_key_data, scm_username]): self.skipTest('no private hg repo defined for ssh!') project = self.create_project( name='my private hg project over ssh', @@ -859,6 +1102,39 @@ class ProjectUpdatesTest(BaseTransactionTest): scm_key_data=scm_key_data, ) self.check_project_scm(project) + # Test project using SSH username/password instead of key. Should fail + # because of bad password, but never hang. + project2 = self.create_project( + name='my other private hg project over ssh', + scm_type='hg', + scm_url=scm_url, + scm_username=scm_username, + scm_password=scm_password, + ) + self.check_project_update(project2, should_fail=True) + + def test_hg_project_from_local_path(self): + # Create temp repository directory. + repo_dir = tempfile.mkdtemp() + self._temp_project_dirs.append(repo_dir) + handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir) + test_playbook_file = os.fdopen(handle, 'w') + test_playbook_file.write(TEST_PLAYBOOK) + test_playbook_file.close() + subprocess.check_call(['hg', 'init', '.'], cwd=repo_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + subprocess.check_call(['hg', 'add', os.path.basename(playbook_path)], + cwd=repo_dir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + subprocess.check_call(['hg', 'commit', '-m', 'blah'], cwd=repo_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # Now test project using local repo. + project = self.create_project( + name='my hg project from local path', + scm_type='hg', + scm_url=repo_dir, + ) + self.check_project_scm(project) def test_public_svn_project_over_https(self): scm_url = getattr(settings, 'TEST_SVN_PUBLIC_HTTPS', @@ -887,6 +1163,30 @@ class ProjectUpdatesTest(BaseTransactionTest): ) self.check_project_scm(project) + def test_svn_project_from_local_path(self): + # Create temp repository directory. + repo_dir = tempfile.mkdtemp() + self._temp_project_dirs.append(repo_dir) + subprocess.check_call(['svnadmin', 'create', '.'], cwd=repo_dir, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir) + test_playbook_file = os.fdopen(handle, 'w') + test_playbook_file.write(TEST_PLAYBOOK) + test_playbook_file.close() + subprocess.check_call(['svn', 'import', '-m', 'blah', + os.path.basename(playbook_path), + 'file://%s/%s' % (repo_dir, os.path.basename(playbook_path))], + cwd=repo_dir, stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + scm_url = 'file://%s' % repo_dir + # Now test project using local repo. + project = self.create_project( + name='my svn project from local path', + scm_type='svn', + scm_url=scm_url, + ) + self.check_project_scm(project) + def test_prompt_for_scm_password_on_update(self): scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS', 'https://github.com/ansible/ansible.github.com.git') diff --git a/awx/main/utils.py b/awx/main/utils.py index 0be324ab11..60c755c322 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -8,6 +8,7 @@ import logging import re import subprocess import sys +import urlparse # Django REST Framework from rest_framework.exceptions import ParseError, PermissionDenied @@ -16,7 +17,7 @@ from rest_framework.exceptions import ParseError, PermissionDenied from Crypto.Cipher import AES __all__ = ['get_object_or_400', 'get_object_or_403', 'camelcase_to_underscore', - 'get_ansible_version', 'get_awx_version'] + 'get_ansible_version', 'get_awx_version', 'update_scm_url'] def get_object_or_400(klass, *args, **kwargs): ''' @@ -126,3 +127,73 @@ def decrypt_field(instance, field_name): cipher = AES.new(key, AES.MODE_ECB) value = cipher.decrypt(encrypted) return value.rstrip('\x00') + +def update_scm_url(scm_type, url, username=True, password=True): + ''' + Update the given SCM URL to add/replace/remove the username/password. When + username/password is True, preserve existing username/password, when + False (None, '', etc.), remove any existing username/password, otherwise + replace username/password. Also validates the given URL. + ''' + # Handle all of the URL formats supported by the SCM systems: + # git: https://www.kernel.org/pub/software/scm/git/docs/git-clone.html#URLS + # hg: http://www.selenic.com/mercurial/hg.1.html#url-paths + # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls + if scm_type not in ('git', 'hg', 'svn'): + raise ValueError('unsupported SCM type "%s"' % str(scm_type)) + parts = urlparse.urlsplit(url) + #print parts + if '://' not in url: + # Handle SCP-style URLs for git (e.g. [user@]host.xz:path/to/repo.git/). + if scm_type == 'git' and '@' in url: + userpass, hostpath = url.split('@', 1) + hostpath = '/'.join(hostpath.split(':', 1)) + modified_url = '@'.join([userpass, hostpath]) + parts = urlparse.urlsplit('ssh://%s' % modified_url) + elif scm_type == 'git' and ':' in url: + modified_url = '/'.join(url.split(':', 1)) + parts = urlparse.urlsplit('ssh://%s' % modified_url) + # Handle local paths specified without file scheme (e.g. /path/to/foo). + # Only supported by git and hg. + elif scm_type in ('git', 'hg'): + if not url.startswith('/'): + parts = urlparse.urlsplit('file:///%s' % url) + else: + parts = urlparse.urlsplit('file://%s' % url) + else: + raise ValueError('unsupported %s URL "%s"' % (scm_type, url)) + #print parts + # Validate that scheme is valid for given scm_type. + scm_type_schemes = { + 'git': ('ssh', 'git', 'http', 'https', 'ftp', 'ftps', 'rsync', 'file'), + 'hg': ('file', 'http', 'https', 'ssh'), + 'svn': ('file', 'http', 'https', 'svn', 'svn+ssh'), + } + if parts.scheme not in scm_type_schemes.get(scm_type, ()): + raise ValueError('unsupported %s scheme "%s"' % (scm_type, parts.scheme)) + if parts.scheme == 'file' and parts.netloc not in ('', 'localhost'): + raise ValueError('unsupported host "%s" for file:// URL' % (parts.netloc)) + elif parts.scheme != 'file' and not parts.netloc: + raise ValueError('host is required for %s URL' % parts.scheme) + if username is True: + netloc_username = parts.username or '' + elif username: + netloc_username = username + else: + netloc_username = '' + if password is True: + netloc_password = parts.password or '' + elif password: + netloc_password = password + else: + netloc_password = '' + if netloc_username and parts.scheme != 'file': + netloc = u':'.join(filter(None, [netloc_username, netloc_password])) + else: + netloc = u'' + netloc = u'@'.join(filter(None, [netloc, parts.hostname])) + if parts.port: + netloc = u':'.join([netloc, unicode(parts.port)]) + new_url = urlparse.urlunsplit([parts.scheme, netloc, parts.path, + parts.query, parts.fragment]) + return new_url