diff --git a/lib/main/models/__init__.py b/lib/main/models/__init__.py index de08b3881d..62ff65d93b 100644 --- a/lib/main/models/__init__.py +++ b/lib/main/models/__init__.py @@ -31,7 +31,6 @@ from djcelery.models import TaskMeta from rest_framework.authtoken.models import Token import yaml -# TODO: jobs and events model TBD # TODO: reporting model TBD PERM_INVENTORY_ADMIN = 'admin' @@ -427,12 +426,11 @@ class Host(CommonModelNameNotUnique): import lib.urls return reverse(lib.urls.views_HostsDetail, args=(self.pk,)) - # relationship to LaunchJobStatus - # relationship to LaunchJobStatusEvent - # last_job_status + # Use .job_host_summaries.all() to get jobs affecting this host. + # Use .job_events.all() to get events affecting this host. + # Use .job_host_summaries.order_by('-pk')[0] to get the last result. class Group(CommonModelNameNotUnique): - ''' A group of managed nodes. May belong to multiple groups ''' @@ -515,33 +513,6 @@ class Credential(CommonModelNameNotUnique): user = models.ForeignKey('auth.User', null=True, default=None, blank=True, on_delete=SET_NULL, related_name='credentials') team = models.ForeignKey('Team', null=True, default=None, blank=True, on_delete=SET_NULL, related_name='credentials') - # IF ssh_key_path is SET - # - # STAGE 1: SSH KEY SUPPORT - # - # ssh-agent bash & - # save keyfile to tempdir in /var/tmp (permissions guarded) - # ssh-add path-to-keydata - # key could locked or unlocked, so use 'expect like' code to enter it at the prompt - # if key is locked: - # if ssh_key_unlock is provided provide key password - # if not provided, FAIL - # - # ssh_username if set corresponds to -u on ansible-playbook, if unset -u root - # - # STAGE 2: - # OR if ssh_password is set instead, do not use SSH agent - # set ANSIBLE_SSH_PASSWORD - # - # STAGE 3: - # - # MICHAEL: modify ansible/ansible-playbook such that - # if ANSIBLE_PASSWORD or ANSIBLE_SUDO_PASSWORD is set - # you do not have to use --ask-pass and --ask-sudo-pass, so we don't have to do interactive - # stuff with that. - # - # ansible-playbook foo.yml ... - ssh_username = models.CharField( blank=True, default='', @@ -753,7 +724,7 @@ class Project(CommonModel): try: if 'hosts' not in data[0] and 'include' not in data[0]: continue - except (IndexError, KeyError): + except (TypeError, IndexError, KeyError): continue playbook = os.path.relpath(playbook, self.local_path) # Filter files in a roles subdirectory. diff --git a/lib/main/tasks.py b/lib/main/tasks.py index 1e2e0d95cb..35fea4e8aa 100644 --- a/lib/main/tasks.py +++ b/lib/main/tasks.py @@ -147,75 +147,6 @@ class RunJob(Task): args = ['ssh-agent', 'sh', '-c', cmd] return args - def capture_subprocess_output(self, proc, timeout=1.0): - ''' - Capture stdout/stderr from the given process until the timeout expires. - ''' - stdout, stderr = '', '' - until = time.time() + timeout - remaining = max(0, until - time.time()) - while remaining > 0: - # FIXME: Probably want to use poll (when on Linux), needs to be tested. - if hasattr(select, 'poll') and False: - poll = select.poll() - poll.register(proc.stdout.fileno(), select.POLLIN or select.POLLPRI) - poll.register(proc.stderr.fileno(), select.POLLIN or select.POLLPRI) - fd_events = poll.poll(remaining) - if not fd_events: - break - for fd, evt in fd_events: - if fd == proc.stdout.fileno() and evt > 0: - stdout += proc.stdout.read(1) - elif fd == proc.stderr.fileno() and evt > 0: - stderr += proc.stderr.read(1) - else: - stdout_byte, stderr_byte = '', '' - fdlist = [proc.stdout.fileno(), proc.stderr.fileno()] - rwx = select.select(fdlist, [], [], remaining) - if proc.stdout.fileno() in rwx[0]: - stdout_byte = proc.stdout.read(1) - stdout += stdout_byte - if proc.stderr.fileno() in rwx[0]: - stderr_byte = proc.stderr.read(1) - stderr += stderr_byte - if not stdout_byte and not stderr_byte: - break - remaining = max(0, until - time.time()) - return stdout, stderr - - def run_subprocess(self, job_pk, args, cwd, env, passwords): - ''' - Run the job using subprocess to capture stdout/stderr. - ''' - status, stdout, stderr = 'error', '', '' - proc = subprocess.Popen(args, cwd=cwd, env=env, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - proc_canceled = False - while proc.poll() is None: - new_stdout, new_stderr = self.capture_subprocess_output(proc) - job_updates = {} - if new_stdout: - stdout += new_stdout - job_updates['result_stdout'] = stdout - if new_stderr: - stderr += new_stderr - job_updates['result_stdout'] = stdout - job = self.update_job(job_pk, **job_updates) - if job.cancel_flag and not proc_canceled: - proc.terminate() - proc_canceled = True - stdout += proc.stdout.read() - stderr += proc.stderr.read() - if proc_canceled: - status = 'canceled' - elif proc.returncode == 0: - status = 'successful' - else: - status = 'failed' - return status, stdout, stderr - def run_pexpect(self, job_pk, args, cwd, env, passwords): ''' Run the job using pexpect to capture output and provide passwords when @@ -273,8 +204,6 @@ class RunJob(Task): args = self.build_args(job, **kwargs) cwd = job.project.local_path env = self.build_env(job, **kwargs) - #status, stdout, stderr = self.run_subprocess(job_pk, args, cwd, - # env, passwords) status, stdout, stderr = self.run_pexpect(job_pk, args, cwd, env, kwargs['passwords']) except Exception: diff --git a/lib/main/tests/commands.py b/lib/main/tests/commands.py index 12cdf1c1b0..5963cac060 100644 --- a/lib/main/tests/commands.py +++ b/lib/main/tests/commands.py @@ -243,7 +243,7 @@ class AcomInventoryTest(BaseCommandTest): def test_with_invalid_inventory_id(self): inventory_pks = set(map(lambda x: x.pk, self.inventories)) - invalid_id = [x for x in xrange(9999) if x not in inventory_pks][0] + invalid_id = [x for x in xrange(1, 9999) if x not in inventory_pks][0] os.environ['ACOM_INVENTORY_ID'] = str(invalid_id) result, stdout, stderr = self.run_command('acom_inventory', list=True) self.assertTrue(isinstance(result, CommandError)) diff --git a/lib/main/tests/projects.py b/lib/main/tests/projects.py index fb6437ae5c..b5c3fe96ac 100644 --- a/lib/main/tests/projects.py +++ b/lib/main/tests/projects.py @@ -23,6 +23,13 @@ from django.test.client import Client from lib.main.models import * from lib.main.tests.base import BaseTest +TEST_PLAYBOOK = '''- hosts: mygroup + gather_facts: false + tasks: + - name: woohoo + command: test 1 = 1 +''' + class ProjectsTest(BaseTest): # tests for users, projects, and teams @@ -93,6 +100,51 @@ class ProjectsTest(BaseTest): # here is a user without any permissions... return ('nobody', 'nobody') + def test_available_playbooks(self): + def write_test_file(project, name, content): + full_path = os.path.join(project.local_path, name) + if not os.path.exists(os.path.dirname(full_path)): + os.makedirs(os.path.dirname(full_path)) + f = file(os.path.join(project.local_path, name), 'wb') + f.write(content) + f.close() + # Invalid local_path + project = self.projects[0] + project.local_path = os.path.join(project.local_path, + 'does_not_exist') + project.save() + self.assertEqual(len(project.available_playbooks), 0) + # Simple playbook + project = self.projects[1] + write_test_file(project, 'foo.yml', TEST_PLAYBOOK) + self.assertEqual(len(project.available_playbooks), 1) + # Other files + project = self.projects[2] + write_test_file(project, 'foo.txt', 'not a playbook') + self.assertEqual(len(project.available_playbooks), 0) + # Empty playbook + project = self.projects[3] + write_test_file(project, 'blah.yml', '') + self.assertEqual(len(project.available_playbooks), 0) + # Invalid YAML + project = self.projects[4] + write_test_file(project, 'blah.yml', TEST_PLAYBOOK + '----') + self.assertEqual(len(project.available_playbooks), 0) + # No hosts or includes + project = self.projects[5] + playbook_content = TEST_PLAYBOOK.replace('hosts', 'hoists') + write_test_file(project, 'blah.yml', playbook_content) + self.assertEqual(len(project.available_playbooks), 0) + # Playbook in roles folder + project = self.projects[6] + write_test_file(project, 'roles/blah.yml', TEST_PLAYBOOK) + self.assertEqual(len(project.available_playbooks), 0) + # Playbook in tasks folder + project = self.projects[7] + write_test_file(project, 'tasks/blah.yml', TEST_PLAYBOOK) + self.assertEqual(len(project.available_playbooks), 0) + + def test_mainline(self): # ===================================================================== diff --git a/lib/main/tests/tasks.py b/lib/main/tests/tasks.py index e79d50b371..ba0aa5c908 100644 --- a/lib/main/tests/tasks.py +++ b/lib/main/tests/tasks.py @@ -135,9 +135,11 @@ class RunJobTest(BaseCeleryTest): # Monkeypatch RunJob to capture list of command line arguments. self.original_build_args = RunJob.build_args self.run_job_args = None + self.build_args_callback = lambda: None def new_build_args(_self, job, **kw): args = self.original_build_args(_self, job, **kw) self.run_job_args = args + self.build_args_callback() return args RunJob.build_args = new_build_args @@ -218,6 +220,9 @@ class RunJobTest(BaseCeleryTest): self.assertEqual(job.status, 'successful') self.assertTrue(job.result_stdout) job_events = job.job_events.all() + for job_event in job_events: + unicode(job_event) # For test coverage. + job_event.save() self.assertEqual(job_events.filter(event='playbook_on_start').count(), 1) self.assertEqual(job_events.filter(event='playbook_on_play_start').count(), 1) self.assertEqual(job_events.filter(event='playbook_on_task_start').count(), 2) @@ -225,6 +230,8 @@ class RunJobTest(BaseCeleryTest): for evt in job_events.filter(event='runner_on_ok'): self.assertEqual(evt.host, self.host) self.assertEqual(job_events.filter(event='playbook_on_stats').count(), 1) + for job_host_summary in job.job_host_summaries.all(): + unicode(job_host_summary) # For test coverage. self.assertEqual(job.successful_hosts.count(), 1) self.assertEqual(job.failed_hosts.count(), 0) self.assertEqual(job.changed_hosts.count(), 1) @@ -310,18 +317,38 @@ class RunJobTest(BaseCeleryTest): self.assertEqual(job.skipped_hosts.count(), 1) self.assertEqual(job.processed_hosts.count(), 1) + def _cancel_job_callback(self): + job = Job.objects.get(pk=self.job.pk) + self.assertTrue(job.cancel()) + self.assertTrue(job.cancel()) # No change from calling again. + def test_cancel_job(self): self.create_test_project(TEST_PLAYBOOK) job_template = self.create_test_job_template() - # The cancel_flag isn't checked until after the job is started, so - # setting it here will allow the job to start, then interrupt it. - job = self.create_test_job(job_template=job_template, cancel_flag=True) + # Pass save=False just for the sake of test coverage. + job = self.create_test_job(job_template=job_template, save=False) + job.save() self.assertEqual(job.status, 'new') + self.assertEqual(job.cancel_flag, False) + # Calling cancel before start has no effect. + self.assertFalse(job.cancel()) + self.assertEqual(job.cancel_flag, False) self.assertFalse(job.get_passwords_needed_to_start()) + self.build_args_callback = self._cancel_job_callback self.assertTrue(job.start()) self.assertEqual(job.status, 'pending') job = Job.objects.get(pk=job.pk) self.assertEqual(job.status, 'canceled') + self.assertEqual(job.cancel_flag, True) + # Calling cancel afterwards just returns the cancel flag. + self.assertTrue(job.cancel()) + # Read attribute for test coverage. + job.celery_task + job.celery_task_id = '' + job.save() + self.assertEqual(job.celery_task, None) + # Unable to start job again. + self.assertFalse(job.start()) def test_extra_job_options(self): self.create_test_project(TEST_PLAYBOOK)