From a0bdf8cdae09f642088417428fdc3d0375f8ce15 Mon Sep 17 00:00:00 2001 From: Jake McDermott Date: Fri, 11 Sep 2020 15:28:49 -0400 Subject: [PATCH] Add default sysjob days --- awx/api/serializers.py | 11 +++- awx/api/views/__init__.py | 7 ++- .../migrations/0124_sysjob_default_days.py | 36 +++++++++++++ awx/main/models/jobs.py | 54 +++++++++++++++---- .../functional/api/test_job_runtime_params.py | 28 +++++++++- awx/main/tests/functional/conftest.py | 5 +- 6 files changed, 127 insertions(+), 14 deletions(-) create mode 100644 awx/main/migrations/0124_sysjob_default_days.py diff --git a/awx/api/serializers.py b/awx/api/serializers.py index ecce831a19..c0d738207b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -3335,9 +3335,11 @@ class AdHocCommandRelaunchSerializer(AdHocCommandSerializer): class SystemJobTemplateSerializer(UnifiedJobTemplateSerializer): + has_configurable_retention = serializers.BooleanField() + class Meta: model = SystemJobTemplate - fields = ('*', 'job_type',) + fields = ('*', 'job_type', 'has_configurable_retention', 'default_days',) def get_related(self, obj): res = super(SystemJobTemplateSerializer, self).get_related(obj) @@ -3348,10 +3350,15 @@ class SystemJobTemplateSerializer(UnifiedJobTemplateSerializer): notification_templates_started = self.reverse('api:system_job_template_notification_templates_started_list', kwargs={'pk': obj.pk}), notification_templates_success = self.reverse('api:system_job_template_notification_templates_success_list', kwargs={'pk': obj.pk}), notification_templates_error = self.reverse('api:system_job_template_notification_templates_error_list', kwargs={'pk': obj.pk}), - )) return res + def to_representation(self, obj): + result = super(SystemJobTemplateSerializer, self).to_representation(obj) + if not obj.has_configurable_retention: + del result['default_days'] + return result + class SystemJobSerializer(UnifiedJobSerializer): diff --git a/awx/api/views/__init__.py b/awx/api/views/__init__.py index 43e845af0c..29b26c120d 100644 --- a/awx/api/views/__init__.py +++ b/awx/api/views/__init__.py @@ -82,6 +82,7 @@ from awx.main.utils import ( get_object_or_400, getattrd, get_pk_from_dict, + parse_yaml_or_json, schedule_task_manager, ignore_inventory_computed_fields, set_environ @@ -3440,7 +3441,11 @@ class SystemJobTemplateLaunch(GenericAPIView): def post(self, request, *args, **kwargs): obj = self.get_object() - new_job = obj.create_unified_job(extra_vars=request.data.get('extra_vars', {})) + extra_vars = parse_yaml_or_json(request.data.get('extra_vars', {})) + if obj.has_configurable_retention and obj.default_days is not None: + extra_vars.setdefault('days', obj.default_days) + + new_job = obj.create_unified_job(extra_vars=extra_vars) new_job.signal_start() data = OrderedDict() data['system_job'] = new_job.id diff --git a/awx/main/migrations/0124_sysjob_default_days.py b/awx/main/migrations/0124_sysjob_default_days.py new file mode 100644 index 0000000000..1bba82b69c --- /dev/null +++ b/awx/main/migrations/0124_sysjob_default_days.py @@ -0,0 +1,36 @@ +# -*- coding: utf-8 -*- + +from django.db import migrations, models + +from awx.main.utils.common import set_current_apps +from awx.main.models import SystemJobTemplate + + +def set_default_days(apps, schema_editor): + set_current_apps(apps) + for sys_template in SystemJobTemplate.objects.all(): + if sys_template.has_configurable_retention: + if sys_template.default_days is None: + sys_template.default_days = 30 + sys_template.save() + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0123_drop_hg_support'), + ] + + operations = [ + migrations.AddField( + model_name='systemjob', + name='default_days', + field=models.PositiveIntegerField(blank=True, default=None, null=True), + ), + migrations.AddField( + model_name='systemjobtemplate', + name='default_days', + field=models.PositiveIntegerField(blank=True, default=None, null=True), + ), + migrations.RunPython(set_default_days), + ] diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 638954e53c..d1c56823bd 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -1142,22 +1142,58 @@ class SystemJobOptions(BaseModel): Common fields for SystemJobTemplate and SystemJob. ''' - SYSTEM_JOB_TYPE = [ - ('cleanup_jobs', _('Remove jobs older than a certain number of days')), - ('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')), - ('cleanup_sessions', _('Removes expired browser sessions from the database')), - ('cleanup_tokens', _('Removes expired OAuth 2 access tokens and refresh tokens')) - ] - class Meta: abstract = True + SYSTEM_CLEANUP_JOBS = ( + 'cleanup_jobs', + _('Remove jobs older than a certain number of days') + ) + SYSTEM_CLEANUP_ACTIVITY = ( + 'cleanup_activitystream', + _('Remove activity stream entries older than a certain number of days') + ) + SYSTEM_CLEANUP_SESSIONS = ( + 'cleanup_sessions', + _('Removes expired browser sessions from the database') + ) + SYSTEM_CLEANUP_TOKENS = ( + 'cleanup_tokens', + _('Removes expired OAuth 2 access tokens and refresh tokens') + ) + + SYSTEM_JOB_TYPES = ( + SYSTEM_CLEANUP_JOBS, + SYSTEM_CLEANUP_ACTIVITY, + SYSTEM_CLEANUP_SESSIONS, + SYSTEM_CLEANUP_TOKENS, + ) + CONFIGURABLE_RETENTION_TYPES = ( + SYSTEM_CLEANUP_JOBS, + SYSTEM_CLEANUP_ACTIVITY, + ) + job_type = models.CharField( max_length=32, - choices=SYSTEM_JOB_TYPE, + choices=SYSTEM_JOB_TYPES, blank=True, default='', ) + default_days = models.PositiveIntegerField( + blank=True, + null=True, + default=None, + ) + + @property + def has_configurable_retention(self): + return self.job_type in (name for (name, _) in SystemJobTemplate.CONFIGURABLE_RETENTION_TYPES) + + def clean_default_days(self): + if not self.has_configurable_retention: + if self.default_days is not None: + raise ValidationError(_(f'Data retention isn\'t configurable for type {self.job_type}')) + return self.default_days class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions): @@ -1221,7 +1257,7 @@ class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions): for key in unallowed_vars: rejected[key] = data.pop(key) - if self.job_type in ('cleanup_jobs', 'cleanup_activitystream'): + if self.has_configurable_retention: if 'days' in data: try: if isinstance(data['days'], (bool, type(None))): diff --git a/awx/main/tests/functional/api/test_job_runtime_params.py b/awx/main/tests/functional/api/test_job_runtime_params.py index 80b2fcadfb..78a1e3ba11 100644 --- a/awx/main/tests/functional/api/test_job_runtime_params.py +++ b/awx/main/tests/functional/api/test_job_runtime_params.py @@ -1,3 +1,4 @@ +from collections import namedtuple from unittest import mock import pytest import yaml @@ -6,7 +7,7 @@ import json from awx.api.serializers import JobLaunchSerializer from awx.main.models.credential import Credential from awx.main.models.inventory import Inventory, Host -from awx.main.models.jobs import Job, JobTemplate, UnifiedJobTemplate +from awx.main.models.jobs import Job, JobTemplate, UnifiedJobTemplate, SystemJob from awx.api.versioning import reverse @@ -696,3 +697,28 @@ def test_callback_extra_var_takes_priority_over_host_name(mocker, get, job_templ r = get(reverse('api:job_template_callback', kwargs={'pk': job_template.pk}), user=admin_user, expect=200) assert not r.data['matching_hosts'] + + +@pytest.mark.django_db +@pytest.mark.parametrize('default_days,launch_vars,expected_vars', [ + (9000, {}, {'days': 9000}), + (None, {}, {}), + (9000, {'days': 9001}, {'days': 9001}), + (None, {'days': 9001}, {'days': 9001}), +]) +def test_system_job_launch_default_retention(system_job_template, admin_user, post, + default_days, launch_vars, expected_vars): + system_job_template.default_days = default_days + system_job_template.save() + launch_url = reverse( + 'api:system_job_template_launch', + kwargs={'pk': system_job_template.pk} + ) + mock_stdout = namedtuple('MockHandle', ['read'])(lambda: '') + with mock.patch.object(SystemJob, 'result_stdout_raw_handle', return_value=mock_stdout): + with mock.patch.object(SystemJob, 'signal_start') as signal_start: + res = post(launch_url, {'extra_vars': launch_vars}, admin_user, expect=201) + signal_start.assert_called() + + res_vars = SystemJob.objects.get(id=res.data['id']).extra_vars + assert json.loads(res_vars) == expected_vars diff --git a/awx/main/tests/functional/conftest.py b/awx/main/tests/functional/conftest.py index 7111950003..9154abae79 100644 --- a/awx/main/tests/functional/conftest.py +++ b/awx/main/tests/functional/conftest.py @@ -747,7 +747,10 @@ def workflow_job_factory(workflow_job_template, admin): @pytest.fixture def system_job_template(): - sys_jt = SystemJobTemplate(name='test-system_job_template', job_type='cleanup_jobs') + sys_jt = SystemJobTemplate( + name='test-system_job_template', + job_type=SystemJobTemplate.SYSTEM_CLEANUP_JOBS[0] + ) sys_jt.save() return sys_jt