AC-432, AC-437. Updated SCM URL validation, add additional tests for SSH URLs.

awx/main/serializers.py

@@ -25,6 +25,7 @@ from rest_framework import serializers
 
 # AWX
 from awx.main.models import *
+from awx.main.utils import update_scm_url
 
 BASE_FIELDS = ('id', 'url', 'related', 'summary_fields', 'created', 'modified',
                'name', 'description')
@@ -299,11 +300,26 @@ class ProjectSerializer(BaseSerializer):
 
     def validate_scm_url(self, attrs, source):
         if self.object:
-            scm_type = attrs.get('scm_type', self.object.scm_type)
+            scm_type = attrs.get('scm_type', self.object.scm_type) or ''
+            scm_username = attrs.get('scm_username', self.object.scm_username) or ''
+            scm_password = attrs.get('scm_password', self.object.scm_password) or ''
         else:
-            scm_type = attrs.get('scm_type', '')
+            scm_type = attrs.get('scm_type', '') or ''
+            scm_username = attrs.get('scm_username', '') or ''
+            scm_password = attrs.get('scm_password', '') or ''
         scm_url = unicode(attrs.get(source, None) or '')
+        try:
+            if scm_username and scm_password:
+                scm_url = update_scm_url(scm_type, scm_url, scm_username,
+                                         '********')
+            elif scm_username:
+                scm_url = update_scm_url(scm_type, scm_url, scm_username)
+            else:
+                scm_url = update_scm_url(scm_type, scm_url)
+        except ValueError, e:
+            raise serializers.ValidationError((e.args or ('Invalid SCM URL',))[0])
         scm_url_parts = urlparse.urlsplit(scm_url)
+        print scm_url_parts
         if scm_type and not any(scm_url_parts):
             raise serializers.ValidationError('SCM URL must be provided')
         return attrs

awx/main/tasks.py

@@ -40,7 +40,6 @@ class BaseTask(Task):
     
     name = None
     model = None
-    idle_timeout = None
 
     def update_model(self, pk, **updates):
         '''
@@ -129,6 +128,9 @@ class BaseTask(Task):
     def build_output_replacements(self, instance, **kwargs):
         return []
 
+    def get_idle_timeout(self):
+        return None
+
     def get_password_prompts(self):
         '''
         Return a dictionary of prompt regular expressions and password lookup
@@ -152,6 +154,7 @@ class BaseTask(Task):
         child.logfile_read = logfile
         canceled = False
         last_stdout_update = time.time()
+        idle_timeout = self.get_idle_timeout()
         expect_list = []
         expect_passwords = {}
         for n, item in enumerate(self.get_password_prompts().items()):
@@ -174,7 +177,7 @@ class BaseTask(Task):
                 canceled = True
             # FIXME: Configurable idle timeout? Find a way to determine if task
             # is hung waiting at a prompt.
-            if self.idle_timeout and (time.time() - last_stdout_update) > self.idle_timeout:
+            if idle_timeout and (time.time() - last_stdout_update) > idle_timeout:
                 child.close(True)
                 canceled = True
         if canceled:
@@ -430,7 +433,6 @@ class RunProjectUpdate(BaseTask):
     
     name = 'run_project_update'
     model = ProjectUpdate
-    #idle_timeout = 30
 
     def build_passwords(self, project_update, **kwargs):
         '''
@@ -463,27 +465,33 @@ class RunProjectUpdate(BaseTask):
         Helper method to build SCM url and extra vars with parameters needed
         for authentication.
         '''
-        # FIXME: May need to pull username/password out of URL in other cases.
         extra_vars = {}
         project = project_update.project
         scm_type = project.scm_type
-        scm_url = project.scm_url
+        scm_url = update_scm_url(scm_type, project.scm_url)
+        scm_url_parts = urlparse.urlsplit(scm_url)
         scm_username = kwargs.get('passwords', {}).get('scm_username', '')
+        scm_username = scm_username or scm_url_parts.username or ''
         scm_password = kwargs.get('passwords', {}).get('scm_password', '')
+        scm_password = scm_password or scm_url_parts.password or ''
         if scm_username and scm_password not in ('ASK', ''):
             if scm_type == 'svn':
+                # FIXME: Need to somehow escape single/double quotes in username/password
                 extra_vars['scm_username'] = scm_username
                 extra_vars['scm_password'] = scm_password
+                scm_url = update_scm_url(scm_type, scm_url, False, False)
+            elif scm_url_parts.scheme == 'ssh':
+                scm_url = update_scm_url(scm_type, scm_url, scm_username, False)
             else:
                 scm_url = update_scm_url(scm_type, scm_url, scm_username,
                                          scm_password)
         elif scm_username:
             if scm_type == 'svn':
                 extra_vars['scm_username'] = scm_username
+                extra_vars['scm_password'] = ''
+                scm_url = update_scm_url(scm_type, scm_url, False, False)
             else:  
-                scm_url = update_scm_url(scm_type, scm_url, scm_username)
+                scm_url = update_scm_url(scm_type, scm_url, scm_username, False)
-        else:
-            scm_url = update_scm_url(scm_type, scm_url)
         return scm_url, extra_vars
 
     def build_args(self, project_update, **kwargs):
@@ -506,6 +514,7 @@ class RunProjectUpdate(BaseTask):
             'scm_clean': project.scm_clean,
             'scm_delete_on_update': scm_delete_on_update,
         })
+        #print extra_vars
         args.extend(['-e', json.dumps(extra_vars)])
         args.append('project_update.yml')
 
@@ -568,11 +577,15 @@ class RunProjectUpdate(BaseTask):
         d.update({
             r'Username for.*:': 'scm_username',
             r'Password for.*:': 'scm_password',
+            r'^Password:\s*?$': 'scm_password',     # SSH prompt for git.
             # FIXME: Configure whether we should auto accept host keys?
             r'Are you sure you want to continue connecting \(yes/no\)\?': 'yes',
         })
         return d
 
+    def get_idle_timeout(self):
+        return getattr(settings, 'PROJECT_UPDATE_IDLE_TIMEOUT', None)
+
     def pre_run_check(self, project_update, **kwargs):
         '''
         Hook for checking project update before running.

awx/main/tests/projects.py

@@ -3,6 +3,7 @@
 
 # Python
 import datetime
+import getpass
 import json
 import os
 import re
@@ -620,7 +621,8 @@ class ProjectsTest(BaseTest):
 
 @override_settings(CELERY_ALWAYS_EAGER=True,
                    CELERY_EAGER_PROPAGATES_EXCEPTIONS=True,
-                   ANSIBLE_TRANSPORT='local')
+                   ANSIBLE_TRANSPORT='local',
+                   PROJECT_UPDATE_IDLE_TIMEOUT=30)
 class ProjectUpdatesTest(BaseTransactionTest):
 
     def setUp(self):
@@ -802,6 +804,29 @@ class ProjectUpdatesTest(BaseTransactionTest):
                                                   password='testpass')
                 self.assertEqual(new_url_up, updated_url)
 
+    def is_public_key_in_authorized_keys(self):
+        auth_keys = set()
+        auth_keys_path = os.path.expanduser('~/.ssh/authorized_keys')
+        if os.path.exists(auth_keys_path):
+            for line in file(auth_keys_path, 'r'):
+                if line.strip():
+                    key = tuple(line.strip().split()[:2])
+                    auth_keys.add(key)
+        pub_keys = set()
+        rsa_key_path = os.path.expanduser('~/.ssh/id_rsa.pub')
+        if os.path.exists(rsa_key_path):
+            for line in file(rsa_key_path, 'r'):
+                if line.strip():
+                    key = tuple(line.strip().split()[:2])
+                    pub_keys.add(key)
+        dsa_key_path = os.path.expanduser('~/.ssh/id_dsa.pub')
+        if os.path.exists(dsa_key_path):
+            for line in file(dsa_key_path, 'r'):
+                if line.strip():
+                    key = tuple(line.strip().split()[:2])
+                    pub_keys.add(key)
+        return bool(auth_keys & pub_keys)
+
     def check_project_update(self, project, should_fail=False, **kwargs):
         pu = kwargs.pop('project_update', None)
         if not pu:
@@ -821,6 +846,9 @@ class ProjectUpdatesTest(BaseTransactionTest):
         self.assertTrue(match, pu.job_args)
         scm_url_in_args = match.groups()[0]
         self.assertFalse(scm_url_in_args.startswith('/'), scm_url_in_args)
+        #return pu
+        # Make sure scm_password doesn't show up anywhere in args or output
+        # from project update.
         scm_password = kwargs.get('scm_password',
                                   decrypt_field(project, 'scm_password'))
         if scm_password not in ('', 'ASK'):
@@ -831,6 +859,8 @@ class ProjectUpdatesTest(BaseTransactionTest):
                              pu.result_stdout)
             self.assertFalse(scm_password in pu.result_traceback,
                              pu.result_traceback)
+        # Make sure scm_key_unlock doesn't show up anywhere in args or output
+        # from project update.
         scm_key_unlock = kwargs.get('scm_key_unlock',
                                     decrypt_field(project, 'scm_key_unlock'))
         if scm_key_unlock not in ('', 'ASK'):
@@ -865,10 +895,14 @@ class ProjectUpdatesTest(BaseTransactionTest):
         self.fail('no file found to change!')
     
     def check_project_scm(self, project):
+        project = Project.objects.get(pk=project.pk)
         project_path = project.get_project_path(check_if_exists=False)
         # If project could be auto-updated on creation, the project dir should
         # already exist, otherwise run an initial checkout.
         if project.scm_type and not project.scm_passwords_needed:
+            self.assertTrue(project.last_update)
+            self.check_project_update(project,
+                                      project_udpate=project.last_update)
             self.assertTrue(os.path.exists(project_path))
         else:
             self.assertFalse(os.path.exists(project_path))
@@ -931,13 +965,18 @@ class ProjectUpdatesTest(BaseTransactionTest):
         self.assertFalse(os.path.exists(untracked_path))
         # Change username/password for private projects and verify the update
         # fails (but doesn't cause the task to hang).
+        scm_url_parts = urlparse.urlsplit(project.scm_url)
         if project.scm_username and project.scm_password not in ('', 'ASK'):
             scm_username = project.scm_username
+            should_still_fail = not (getpass.getuser() == scm_username and
+                                     scm_url_parts.hostname == 'localhost' and
+                                     'ssh' in scm_url_parts.scheme and
+                                     self.is_public_key_in_authorized_keys())
             # Clear username only.
             project = Project.objects.get(pk=project.pk)
             project.scm_username = ''
             project.save()
-            self.check_project_update(project, should_fail=True)
+            self.check_project_update(project, should_fail=should_still_fail)
             # Try invalid username.
             project = Project.objects.get(pk=project.pk)
             project.scm_username = 'not a\\ valid\' user" name'
@@ -948,17 +987,20 @@ class ProjectUpdatesTest(BaseTransactionTest):
             project.scm_username = ''
             project.scm_password = ''
             project.save()
-            self.check_project_update(project, should_fail=True)
+            self.check_project_update(project, should_fail=should_still_fail)
             # Set username, but no password.
             project = Project.objects.get(pk=project.pk)
             project.scm_username = scm_username
             project.save()
-            self.check_project_update(project, should_fail=True)
+            self.check_project_update(project, should_fail=should_still_fail)
             # Set username, with invalid password.
             project = Project.objects.get(pk=project.pk)
             project.scm_password = 'not a\\ valid\' "password'
             project.save()
-            self.check_project_update(project, should_fail=True)
+            if project.scm_type == 'svn':
+                self.check_project_update(project, should_fail=True)#should_still_fail)
+            else:
+                self.check_project_update(project, should_fail=should_still_fail)
 
     def test_public_git_project_over_https(self):
         scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS',
@@ -1025,8 +1067,7 @@ class ProjectUpdatesTest(BaseTransactionTest):
         )
         self.check_project_update(project2, should_fail=True)
 
-    def test_git_project_from_local_path(self):
+    def create_local_git_repo(self):
-        # Create temp repository directory.
         repo_dir = tempfile.mkdtemp()
         self._temp_project_dirs.append(repo_dir)
         handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir)
@@ -1040,7 +1081,10 @@ class ProjectUpdatesTest(BaseTransactionTest):
                               stderr=subprocess.PIPE)
         subprocess.check_call(['git', 'commit', '-m', 'blah'], cwd=repo_dir,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        # Now test project using local repo.
+        return repo_dir
+
+    def test_git_project_from_local_path(self):
+        repo_dir = self.create_local_git_repo()
         project = self.create_project(
             name='my git project from local path',
             scm_type='git',
@@ -1048,6 +1092,22 @@ class ProjectUpdatesTest(BaseTransactionTest):
         )
         self.check_project_scm(project)
 
+    def test_git_project_via_ssh_loopback(self):
+        scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '')
+        scm_password = getattr(settings, 'TEST_SSH_LOOPBACK_PASSWORD', '')
+        if not all([scm_username, scm_password]):
+            self.skipTest('no ssh loopback username/password defined!')
+        repo_dir = self.create_local_git_repo()
+        scm_url = 'ssh://localhost%s' % repo_dir
+        project = self.create_project(
+            name='my git project via ssh loopback',
+            scm_type='git',
+            scm_url=scm_url,
+            scm_username=scm_username,
+            scm_password=scm_password,
+        )
+        self.check_project_scm(project)
+
     def test_public_hg_project_over_https(self):
         scm_url = getattr(settings, 'TEST_HG_PUBLIC_HTTPS',
                           'https://bitbucket.org/cchurch/django-hotrunner')
@@ -1091,9 +1151,7 @@ class ProjectUpdatesTest(BaseTransactionTest):
     def test_private_hg_project_over_ssh(self):
         scm_url = getattr(settings, 'TEST_HG_PRIVATE_SSH', '')
         scm_key_data = getattr(settings, 'TEST_HG_KEY_DATA', '')
-        scm_username = getattr(settings, 'TEST_HG_USERNAME', '')
+        if not all([scm_url, scm_key_data]):
-        scm_password = 'blahblahblah'
-        if not all([scm_url, scm_key_data, scm_username]):
             self.skipTest('no private hg repo defined for ssh!')
         project = self.create_project(
             name='my private hg project over ssh',
@@ -1102,19 +1160,9 @@ class ProjectUpdatesTest(BaseTransactionTest):
             scm_key_data=scm_key_data,
         )
         self.check_project_scm(project)
-        # Test project using SSH username/password instead of key. Should fail
+        # hg doesn't support password for ssh:// urls.
-        # because of bad password, but never hang.
-        project2 = self.create_project(
-            name='my other private hg project over ssh',
-            scm_type='hg',
-            scm_url=scm_url,
-            scm_username=scm_username,
-            scm_password=scm_password,
-        )
-        self.check_project_update(project2, should_fail=True)
 
-    def test_hg_project_from_local_path(self):
+    def create_local_hg_repo(self):
-        # Create temp repository directory.
         repo_dir = tempfile.mkdtemp()
         self._temp_project_dirs.append(repo_dir)
         handle, playbook_path = tempfile.mkstemp(suffix='.yml', dir=repo_dir)
@@ -1128,7 +1176,10 @@ class ProjectUpdatesTest(BaseTransactionTest):
                               stderr=subprocess.PIPE)
         subprocess.check_call(['hg', 'commit', '-m', 'blah'], cwd=repo_dir,
                               stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        # Now test project using local repo.
+        return repo_dir
+
+    def test_hg_project_from_local_path(self):
+        repo_dir = self.create_local_hg_repo()
         project = self.create_project(
             name='my hg project from local path',
             scm_type='hg',
@@ -1136,6 +1187,23 @@ class ProjectUpdatesTest(BaseTransactionTest):
         )
         self.check_project_scm(project)
 
+    def _test_hg_project_via_ssh_loopback(self):
+        # hg doesn't support password for ssh:// urls.
+        scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '')
+        if not all([scm_username]):
+            self.skipTest('no ssh loopback username defined!')
+        if not self.is_public_key_in_authorized_keys():
+            self.skipTest('ssh loopback for hg requires public key in authorized keys')
+        repo_dir = self.create_local_hg_repo()
+        scm_url = 'ssh://localhost/%s' % repo_dir
+        project = self.create_project(
+            name='my hg project via ssh loopback',
+            scm_type='hg',
+            scm_url=scm_url,
+            scm_username=scm_username,
+        )
+        self.check_project_scm(project)
+
     def test_public_svn_project_over_https(self):
         scm_url = getattr(settings, 'TEST_SVN_PUBLIC_HTTPS',
                           'https://github.com/ansible/ansible.github.com')
@@ -1163,8 +1231,7 @@ class ProjectUpdatesTest(BaseTransactionTest):
         )
         self.check_project_scm(project)
 
-    def test_svn_project_from_local_path(self):
+    def create_local_svn_repo(self):
-        # Create temp repository directory.
         repo_dir = tempfile.mkdtemp()
         self._temp_project_dirs.append(repo_dir)
         subprocess.check_call(['svnadmin', 'create', '.'], cwd=repo_dir,
@@ -1178,8 +1245,11 @@ class ProjectUpdatesTest(BaseTransactionTest):
                                'file://%s/%s' % (repo_dir, os.path.basename(playbook_path))],
                               cwd=repo_dir, stdout=subprocess.PIPE,
                               stderr=subprocess.PIPE)
+        return repo_dir
+
+    def test_svn_project_from_local_path(self):
+        repo_dir = self.create_local_svn_repo()
         scm_url = 'file://%s' % repo_dir
-        # Now test project using local repo.
         project = self.create_project(
             name='my svn project from local path',
             scm_type='svn',
@@ -1187,6 +1257,22 @@ class ProjectUpdatesTest(BaseTransactionTest):
         )
         self.check_project_scm(project)
 
+    def test_svn_project_via_ssh_loopback(self):
+        scm_username = getattr(settings, 'TEST_SSH_LOOPBACK_USERNAME', '')
+        scm_password = getattr(settings, 'TEST_SSH_LOOPBACK_PASSWORD', '')
+        if not all([scm_username, scm_password]):
+            self.skipTest('no ssh loopback username/password defined!')
+        repo_dir = self.create_local_svn_repo()
+        scm_url = 'svn+ssh://localhost%s' % repo_dir
+        project = self.create_project(
+            name='my svn project via ssh loopback',
+            scm_type='svn',
+            scm_url=scm_url,
+            scm_username=scm_username,
+            scm_password=scm_password,
+        )
+        self.check_project_scm(project)
+
     def test_prompt_for_scm_password_on_update(self):
         scm_url = getattr(settings, 'TEST_GIT_PUBLIC_HTTPS',
                           'https://github.com/ansible/ansible.github.com.git')

awx/main/utils.py

@@ -141,6 +141,8 @@ def update_scm_url(scm_type, url, username=True, password=True):
     # svn: http://svnbook.red-bean.com/en/1.7/svn-book.html#svn.advanced.reposurls
     if scm_type not in ('git', 'hg', 'svn'):
         raise ValueError('unsupported SCM type "%s"' % str(scm_type))
+    if not url.strip():
+        return ''
     parts = urlparse.urlsplit(url)
     #print parts
     if '://' not in url:

awx/settings/local_settings.py.example

@@ -260,6 +260,11 @@ TEST_SVN_PASSWORD = ''
 TEST_SVN_PUBLIC_HTTPS = 'https://github.com/ansible/ansible-examples'
 TEST_SVN_PRIVATE_HTTPS = 'https://github.com/ansible/ansible-doc'
 
+# To test repo access via SSH login to localhost.
+import getpass
+TEST_SSH_LOOPBACK_USERNAME = getpass.getuser()
+TEST_SSH_LOOPBACK_PASSWORD = ''
+
 ###############################################################################
 # LDAP TEST SETTINGS
 ###############################################################################