Enable canceling on certain levels.... fix up some unit tests

This commit is contained in:
Matthew Jones
2014-01-30 10:54:54 -05:00
parent 6afc584a4f
commit 280527f993
4 changed files with 30 additions and 33 deletions

View File

@@ -343,7 +343,6 @@ class Job(CommonTask):
opts = dict([(field, kwargs.get(field, '')) for field in needed]) opts = dict([(field, kwargs.get(field, '')) for field in needed])
if not all(opts.values()): if not all(opts.values()):
return False return False
# TODO: This is temporary to allow a dependent task to continue
self.status = 'waiting' self.status = 'waiting'
self.save(update_fields=['status']) self.save(update_fields=['status'])
transaction.commit() transaction.commit()

View File

@@ -237,12 +237,10 @@ class BaseTask(Task):
if logfile_pos != logfile.tell(): if logfile_pos != logfile.tell():
logfile_pos = logfile.tell() logfile_pos = logfile.tell()
last_stdout_update = time.time() last_stdout_update = time.time()
#TODO: Find replacement for cancel flag # NOTE: In case revoke doesn't have an affect
#TODO: Something about checking celery status if instance.cancel_flag:
# if instance.cancel_flag: child.close(True)
# child.close(True) canceled = True
# canceled = True
# FIXME: Find a way to determine if task is hung waiting at a prompt.
if idle_timeout and (time.time() - last_stdout_update) > idle_timeout: if idle_timeout and (time.time() - last_stdout_update) > idle_timeout:
child.close(True) child.close(True)
canceled = True canceled = True

View File

@@ -377,7 +377,7 @@ class CleanupJobsTest(BaseCommandMixin, BaseLiveServerTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.assertEqual(job.status, 'successful') self.assertEqual(job.status, 'successful')
# With days=1, no jobs will be deleted. # With days=1, no jobs will be deleted.

View File

@@ -404,7 +404,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.check_job_events(job, 'ok', 1, 2) self.check_job_events(job, 'ok', 1, 2)
@@ -433,7 +433,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.check_job_events(job, 'skipped', 1, 2) self.check_job_events(job, 'skipped', 1, 2)
@@ -461,7 +461,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'failed') self.check_job_result(job, 'failed')
self.check_job_events(job, 'failed', 1, 1) self.check_job_events(job, 'failed', 1, 1)
@@ -489,7 +489,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.check_job_events(job, 'ok', 1, 1, check_ignore_errors=True) self.check_job_events(job, 'ok', 1, 1, check_ignore_errors=True)
@@ -612,7 +612,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
# Since we don't actually run the task, the --check should indicate # Since we don't actually run the task, the --check should indicate
# everything is successful. # everything is successful.
@@ -653,7 +653,7 @@ class RunJobTest(BaseCeleryTest):
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.build_args_callback = self._cancel_job_callback self.build_args_callback = self._cancel_job_callback
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'canceled') self.check_job_result(job, 'canceled')
self.assertEqual(job.cancel_flag, True) self.assertEqual(job.cancel_flag, True)
@@ -676,7 +676,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('--forks=3' in self.run_job_args) self.assertTrue('--forks=3' in self.run_job_args)
@@ -687,7 +687,7 @@ class RunJobTest(BaseCeleryTest):
job2 = self.create_test_job(job_template=job_template2) job2 = self.create_test_job(job_template=job_template2)
self.assertEqual(job2.status, 'new') self.assertEqual(job2.status, 'new')
self.assertTrue(job2.start()) self.assertTrue(job2.start())
self.assertEqual(job2.status, 'pending') self.assertEqual(job2.status, 'waiting')
job2 = Job.objects.get(pk=job2.pk) job2 = Job.objects.get(pk=job2.pk)
self.check_job_result(job2, 'successful') self.check_job_result(job2, 'successful')
# Test with extra_vars as YAML (should be converted to JSON in args). # Test with extra_vars as YAML (should be converted to JSON in args).
@@ -695,7 +695,7 @@ class RunJobTest(BaseCeleryTest):
job3 = self.create_test_job(job_template=job_template3) job3 = self.create_test_job(job_template=job_template3)
self.assertEqual(job3.status, 'new') self.assertEqual(job3.status, 'new')
self.assertTrue(job3.start()) self.assertTrue(job3.start())
self.assertEqual(job3.status, 'pending') self.assertEqual(job3.status, 'waiting')
job3 = Job.objects.get(pk=job3.pk) job3 = Job.objects.get(pk=job3.pk)
self.check_job_result(job3, 'successful') self.check_job_result(job3, 'successful')
@@ -707,7 +707,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.assertTrue(len(job.job_args) > 1024) self.assertTrue(len(job.job_args) > 1024)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
@@ -720,7 +720,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'failed') self.check_job_result(job, 'failed')
self.assertTrue('-l' in self.run_job_args) self.assertTrue('-l' in self.run_job_args)
@@ -733,7 +733,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('ssh-agent' in self.run_job_args) self.assertTrue('ssh-agent' in self.run_job_args)
@@ -746,7 +746,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('-u' in self.run_job_args) self.assertTrue('-u' in self.run_job_args)
@@ -763,7 +763,7 @@ class RunJobTest(BaseCeleryTest):
self.assertFalse(job.start()) self.assertFalse(job.start())
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertTrue(job.start(ssh_password='sshpass')) self.assertTrue(job.start(ssh_password='sshpass'))
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('--ask-pass' in self.run_job_args) self.assertTrue('--ask-pass' in self.run_job_args)
@@ -777,7 +777,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
# Job may fail if current user doesn't have password-less sudo # Job may fail if current user doesn't have password-less sudo
# privileges, but we're mainly checking the command line arguments. # privileges, but we're mainly checking the command line arguments.
@@ -796,7 +796,7 @@ class RunJobTest(BaseCeleryTest):
self.assertFalse(job.start()) self.assertFalse(job.start())
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertTrue(job.start(sudo_password='sudopass')) self.assertTrue(job.start(sudo_password='sudopass'))
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
# Job may fail if current user doesn't have password-less sudo # Job may fail if current user doesn't have password-less sudo
# privileges, but we're mainly checking the command line arguments. # privileges, but we're mainly checking the command line arguments.
@@ -811,7 +811,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('ssh-agent' in self.run_job_args) self.assertTrue('ssh-agent' in self.run_job_args)
@@ -825,7 +825,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('ssh-agent' in self.run_job_args) self.assertTrue('ssh-agent' in self.run_job_args)
@@ -840,7 +840,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'failed') self.check_job_result(job, 'failed')
self.assertTrue('ssh-agent' in self.run_job_args) self.assertTrue('ssh-agent' in self.run_job_args)
@@ -858,7 +858,7 @@ class RunJobTest(BaseCeleryTest):
self.assertFalse(job.start()) self.assertFalse(job.start())
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertTrue(job.start(ssh_key_unlock=TEST_SSH_KEY_DATA_UNLOCK)) self.assertTrue(job.start(ssh_key_unlock=TEST_SSH_KEY_DATA_UNLOCK))
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue('ssh-agent' in self.run_job_args) self.assertTrue('ssh-agent' in self.run_job_args)
@@ -882,7 +882,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.assertTrue(env_var1 in job.job_env) self.assertTrue(env_var1 in job.job_env)
@@ -901,7 +901,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.check_job_events(job, 'ok', 1, 1, async=True) self.check_job_events(job, 'ok', 1, 1, async=True)
@@ -929,7 +929,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'failed') self.check_job_result(job, 'failed')
self.check_job_events(job, 'failed', 1, 1, async=True) self.check_job_events(job, 'failed', 1, 1, async=True)
@@ -957,7 +957,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'failed') self.check_job_result(job, 'failed')
self.check_job_events(job, 'failed', 1, 1, async=True, self.check_job_events(job, 'failed', 1, 1, async=True,
@@ -986,7 +986,7 @@ class RunJobTest(BaseCeleryTest):
self.assertEqual(job.status, 'new') self.assertEqual(job.status, 'new')
self.assertFalse(job.passwords_needed_to_start) self.assertFalse(job.passwords_needed_to_start)
self.assertTrue(job.start()) self.assertTrue(job.start())
self.assertEqual(job.status, 'pending') self.assertEqual(job.status, 'waiting')
job = Job.objects.get(pk=job.pk) job = Job.objects.get(pk=job.pk)
self.check_job_result(job, 'successful') self.check_job_result(job, 'successful')
self.check_job_events(job, 'ok', 1, 1, async=True, async_nowait=True) self.check_job_events(job, 'ok', 1, 1, async=True, async_nowait=True)