From 8ca0c1b9926c079890b79f18b0b08bca62f721a1 Mon Sep 17 00:00:00 2001 From: Christian Adams Date: Wed, 1 May 2019 12:16:15 -0400 Subject: [PATCH] Add clearsessions and cleartokens system jobs * add system job for gathering insights analytics * enforce schedule enablement with analytics setting * remove celery beat analytics job * keep analytics schedule & setting enablement in sync in API * handles updating schedules for multiple sys job schedules * add analytics setting & schedule test * rm ui modal from collection sys job --- awx/api/templates/api/_schedule_detail.md | 2 +- .../api/system_job_template_launch.md | 2 +- .../commands/cleanup_activitystream.py | 2 +- .../management/commands/cleanup_sessions.py | 25 ++++ .../management/commands/cleanup_tokens.py | 28 ++++ .../0078_v360_clear_sessions_tokens_jt.py | 29 ++++ awx/main/migrations/_create_system_jobs.py | 82 +++++++++++ awx/main/models/jobs.py | 3 +- .../management-jobs/card/card.controller.js | 134 ++++++++++-------- 9 files changed, 247 insertions(+), 60 deletions(-) create mode 100644 awx/main/management/commands/cleanup_sessions.py create mode 100644 awx/main/management/commands/cleanup_tokens.py create mode 100644 awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py create mode 100644 awx/main/migrations/_create_system_jobs.py diff --git a/awx/api/templates/api/_schedule_detail.md b/awx/api/templates/api/_schedule_detail.md index 4acd6a5288..04eecb4f23 100644 --- a/awx/api/templates/api/_schedule_detail.md +++ b/awx/api/templates/api/_schedule_detail.md @@ -5,7 +5,7 @@ The following lists the expected format and details of our rrules: * INTERVAL is required * SECONDLY is not supported * TZID is not supported -* RRULE must preceed the rule statements +* RRULE must precede the rule statements * BYDAY is supported but not BYDAY with a numerical prefix * BYYEARDAY and BYWEEKNO are not supported * Only one rrule statement per schedule is supported diff --git a/awx/api/templates/api/system_job_template_launch.md b/awx/api/templates/api/system_job_template_launch.md index ee05d38cb9..80315c6869 100644 --- a/awx/api/templates/api/system_job_template_launch.md +++ b/awx/api/templates/api/system_job_template_launch.md @@ -3,7 +3,7 @@ Launch a Job Template: Make a POST request to this resource to launch the system job template. Variables specified inside of the parameter `extra_vars` are passed to the -system job task as command line parameters. These tasks can be ran manually +system job task as command line parameters. These tasks can be run manually on the host system via the `awx-manage` command. For example on `cleanup_jobs` and `cleanup_activitystream`: diff --git a/awx/main/management/commands/cleanup_activitystream.py b/awx/main/management/commands/cleanup_activitystream.py index c9a306164b..ac790a2e1b 100644 --- a/awx/main/management/commands/cleanup_activitystream.py +++ b/awx/main/management/commands/cleanup_activitystream.py @@ -59,7 +59,7 @@ class Command(BaseCommand): if len(pks_to_delete): ActivityStream.objects.filter(pk__in=pks_to_delete).delete() n_deleted_items += len(pks_to_delete) - self.logger.log(99, "Removed %d items", n_deleted_items) + self.logger.info("Removed {} items".format(n_deleted_items)) def handle(self, *args, **options): self.verbosity = int(options.get('verbosity', 1)) diff --git a/awx/main/management/commands/cleanup_sessions.py b/awx/main/management/commands/cleanup_sessions.py new file mode 100644 index 0000000000..3f4fc16912 --- /dev/null +++ b/awx/main/management/commands/cleanup_sessions.py @@ -0,0 +1,25 @@ +import logging +from django.core import management +from django.core.management.base import BaseCommand + +from django.contrib.sessions.models import Session + + +class Command(BaseCommand): + + def init_logging(self): + log_levels = dict(enumerate([logging.ERROR, logging.INFO, + logging.DEBUG, 0])) + self.logger = logging.getLogger('awx.main.commands.cleanup_sessions') + self.logger.setLevel(log_levels.get(self.verbosity, 0)) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + self.logger.addHandler(handler) + self.logger.propagate = False + + def execute(self, *args, **options): + self.verbosity = int(options.get('verbosity', 1)) + self.init_logging() + total_sessions = Session.objects.all().count() + management.call_command('clearsessions') + self.logger.info("Expired Sessions deleted {}".format(total_sessions - Session.objects.all().count())) diff --git a/awx/main/management/commands/cleanup_tokens.py b/awx/main/management/commands/cleanup_tokens.py new file mode 100644 index 0000000000..c5dfde145f --- /dev/null +++ b/awx/main/management/commands/cleanup_tokens.py @@ -0,0 +1,28 @@ +import logging +from django.core import management +from django.core.management.base import BaseCommand + +from awx.main.models import OAuth2AccessToken +from oauth2_provider.models import RefreshToken + + +class Command(BaseCommand): + + def init_logging(self): + log_levels = dict(enumerate([logging.ERROR, logging.INFO, + logging.DEBUG, 0])) + self.logger = logging.getLogger('awx.main.commands.cleanup_tokens') + self.logger.setLevel(log_levels.get(self.verbosity, 0)) + handler = logging.StreamHandler() + handler.setFormatter(logging.Formatter('%(message)s')) + self.logger.addHandler(handler) + self.logger.propagate = False + + def execute(self, *args, **options): + self.verbosity = int(options.get('verbosity', 1)) + self.init_logging() + total_accesstokens = OAuth2AccessToken.objects.all().count() + total_refreshtokens = RefreshToken.objects.all().count() + management.call_command('cleartokens') + self.logger.info("Expired OAuth 2 Access Tokens deleted: {}".format(total_accesstokens - OAuth2AccessToken.objects.all().count())) + self.logger.info("Expired OAuth 2 Refresh Tokens deleted: {}".format(total_refreshtokens - RefreshToken.objects.all().count())) diff --git a/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py b/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py new file mode 100644 index 0000000000..83095d2056 --- /dev/null +++ b/awx/main/migrations/0078_v360_clear_sessions_tokens_jt.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11.20 on 2019-05-08 14:51 +from __future__ import unicode_literals + +from django.db import migrations, models +from awx.main.migrations._create_system_jobs import create_clearsessions_jt, create_cleartokens_jt + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0077_v360_add_default_orderings'), + ] + + operations = [ + # Schedule Analytics System Job Template + migrations.RunPython(create_clearsessions_jt, migrations.RunPython.noop), + migrations.RunPython(create_cleartokens_jt, migrations.RunPython.noop), + migrations.AlterField( + model_name='systemjob', + name='job_type', + field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('clearsessions', 'Removes expired browser sessions from the database'), ('cleartokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32), + ), + migrations.AlterField( + model_name='systemjobtemplate', + name='job_type', + field=models.CharField(blank=True, choices=[('cleanup_jobs', 'Remove jobs older than a certain number of days'), ('cleanup_activitystream', 'Remove activity stream entries older than a certain number of days'), ('clearsessions', 'Removes expired browser sessions from the database'), ('cleartokens', 'Removes expired OAuth 2 access tokens and refresh tokens')], default='', max_length=32), + ), + ] diff --git a/awx/main/migrations/_create_system_jobs.py b/awx/main/migrations/_create_system_jobs.py new file mode 100644 index 0000000000..6294fa7799 --- /dev/null +++ b/awx/main/migrations/_create_system_jobs.py @@ -0,0 +1,82 @@ +import random +import logging + +from django.db import migrations, models +from django.utils.timezone import now, timedelta + +logger = logging.getLogger('awx.main.migrations') + +__all__ = ['create_collection_jt', 'create_clearsessions_jt', 'create_cleartokens_jt'] + +''' +These methods are called by migrations to create various system job templates + +Create default system job templates if not present. Create default schedules +only if new system job templates were created (i.e. new database). +''' + + +def create_clearsessions_jt(apps, schema_editor): + + SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate') + Schedule = apps.get_model('main', 'Schedule') + ContentType = apps.get_model('contenttypes', 'ContentType') + sjt_ct = ContentType.objects.get_for_model(SystemJobTemplate) + now_dt = now() + schedule_time = now_dt.strftime('%Y%m%dT%H%M%SZ') + + sjt, created = SystemJobTemplate.objects.get_or_create( + job_type='cleanup_sessions', + defaults=dict( + name='Cleanup Expired Sessions', + description='Cleans out expired browser sessions', + polymorphic_ctype=sjt_ct, + created=now_dt, + modified=now_dt, + ), + ) + if created: + sched = Schedule( + name='Cleanup Expired Sessions', + rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1;COUNT=1' % schedule_time, + description='Cleans out expired browser sessions', + enabled=True, + created=now_dt, + modified=now_dt, + extra_data={}, + ) + sched.unified_job_template = sjt + sched.save() + + +def create_cleartokens_jt(apps, schema_editor): + + SystemJobTemplate = apps.get_model('main', 'SystemJobTemplate') + Schedule = apps.get_model('main', 'Schedule') + ContentType = apps.get_model('contenttypes', 'ContentType') + sjt_ct = ContentType.objects.get_for_model(SystemJobTemplate) + now_dt = now() + schedule_time = now_dt.strftime('%Y%m%dT%H%M%SZ') + + sjt, created = SystemJobTemplate.objects.get_or_create( + job_type='cleanup_tokens', + defaults=dict( + name='Cleanup Expired OAuth 2 Tokens', + description='Cleanup expired OAuth 2 access and refresh tokens', + polymorphic_ctype=sjt_ct, + created=now_dt, + modified=now_dt, + ), + ) + if created: + sched = Schedule( + name='Cleanup Expired OAuth 2 Tokens', + rrule='DTSTART:%s RRULE:FREQ=WEEKLY;INTERVAL=1;COUNT=1' % schedule_time, + description='Removes expired OAuth 2 access and refresh tokens', + enabled=True, + created=now_dt, + modified=now_dt, + extra_data={}, + ) + sched.unified_job_template = sjt + sched.save() diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index 60b3c92132..27b4b28394 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -1139,7 +1139,8 @@ class SystemJobOptions(BaseModel): SYSTEM_JOB_TYPE = [ ('cleanup_jobs', _('Remove jobs older than a certain number of days')), ('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')), - ('cleanup_facts', _('Purge and/or reduce the granularity of system tracking data')), + ('clearsessions', _('Removes expired browser sessions from the database')), + ('cleartokens', _('Removes expired OAuth 2 access tokens and refresh tokens')) ] class Meta: diff --git a/awx/ui/client/src/management-jobs/card/card.controller.js b/awx/ui/client/src/management-jobs/card/card.controller.js index 5b6bff4f44..930f04cd51 100644 --- a/awx/ui/client/src/management-jobs/card/card.controller.js +++ b/awx/ui/client/src/management-jobs/card/card.controller.js @@ -45,69 +45,91 @@ export default }); }; + var launchManagementJob = function (defaultUrl){ + var data = {}; + Rest.setUrl(defaultUrl); + Rest.post(data) + .then(({data}) => { + Wait('stop'); + $state.go('output', { id: data.system_job, type: 'system' }, { reload: true }); + }) + .catch(({data, status}) => { + let template_id = $scope.job_template_id; + template_id = (template_id === undefined) ? "undefined" : i18n.sprintf("%d", template_id); + ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), + msg: i18n.sprintf(i18n._('Failed updating job %s with variables. POST returned: %d'), template_id, status) }); + }); + }; + $scope.submitJob = function (id, name) { Wait('start'); defaultUrl = GetBasePath('system_job_templates')+id+'/launch/'; - CreateDialog({ - id: 'prompt-for-days' , - title: name, - scope: $scope, - width: 500, - height: 300, - minWidth: 200, - callback: 'PromptForDays', - resizable: false, - onOpen: function(){ - $scope.$watch('prompt_for_days_form.$invalid', function(invalid) { - if (invalid === true) { - $('#prompt-for-days-launch').prop("disabled", true); - } else { - $('#prompt-for-days-launch').prop("disabled", false); - } - }); + var noModalJobs = ['Cleanup Expired Sessions', 'Cleanup Expired OAuth 2 Tokens']; + if (noModalJobs.includes(name)) { + launchManagementJob(defaultUrl, name); + } else { + + CreateDialog({ + id: 'prompt-for-days', + title: name, + scope: $scope, + width: 500, + height: 300, + minWidth: 200, + callback: 'PromptForDays', + resizable: false, + onOpen: function(){ + $scope.$watch('prompt_for_days_form.$invalid', function(invalid) { + if (invalid === true) { + $('#prompt-for-days-launch').prop("disabled", true); + } else { + $('#prompt-for-days-launch').prop("disabled", false); + } + }); - let fieldScope = $scope.$parent; - fieldScope.days_to_keep = 30; - $scope.prompt_for_days_form.$setPristine(); - $scope.prompt_for_days_form.$invalid = false; - }, - buttons: [ - { - "label": "Cancel", - "onClick": function() { - $(this).dialog('close'); + let fieldScope = $scope.$parent; + fieldScope.days_to_keep = 30; + $scope.prompt_for_days_form.$setPristine(); + $scope.prompt_for_days_form.$invalid = false; + }, + buttons: [ + { + "label": "Cancel", + "onClick": function() { + $(this).dialog('close'); + }, + "class": "btn btn-default", + "id": "prompt-for-days-cancel" }, - "class": "btn btn-default", - "id": "prompt-for-days-cancel" - }, - { - "label": "Launch", - "onClick": function() { - const extra_vars = {"days": $scope.days_to_keep }, - data = {}; - data.extra_vars = JSON.stringify(extra_vars); + { + "label": "Launch", + "onClick": function() { + const extra_vars = {"days": $scope.days_to_keep }, + data = {}; + data.extra_vars = JSON.stringify(extra_vars); - Rest.setUrl(defaultUrl); - Rest.post(data) - .then(({data}) => { - Wait('stop'); - $("#prompt-for-days").dialog("close"); - // $("#configure-dialog").dialog('close'); - $state.go('output', { id: data.system_job, type: 'system' }, { reload: true }); - }) - .catch(({data, status}) => { - let template_id = $scope.job_template_id; - template_id = (template_id === undefined) ? "undefined" : i18n.sprintf("%d", template_id); - ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), - msg: i18n.sprintf(i18n._('Failed updating job %s with variables. POST returned: %d'), template_id, status) }); - }); - }, - "class": "btn btn-primary", - "id": "prompt-for-days-launch" - } - ] - }); + Rest.setUrl(defaultUrl); + Rest.post(data) + .then(({data}) => { + Wait('stop'); + $("#prompt-for-days").dialog("close"); + // $("#configure-dialog").dialog('close'); + $state.go('output', { id: data.system_job, type: 'system' }, { reload: true }); + }) + .catch(({data, status}) => { + let template_id = $scope.job_template_id; + template_id = (template_id === undefined) ? "undefined" : i18n.sprintf("%d", template_id); + ProcessErrors($scope, data, status, null, { hdr: i18n._('Error!'), + msg: i18n.sprintf(i18n._('Failed updating job %s with variables. POST returned: %d'), template_id, status) }); + }); + }, + "class": "btn btn-primary", + "id": "prompt-for-days-launch" + } + ] + }); + } if ($scope.removePromptForDays) { $scope.removePromptForDays();