From 15906b7e3c6910ce5d9f5dd0464677c7cb723bb9 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 19 Jan 2018 11:11:21 -0500 Subject: [PATCH] support TZID= in schedule rrules this commit allows schedule `rrule` strings to include local timezone information via TZID=NNNNN; occurrences are _generated_ in the local time specific by the user (or UTC, if e.g., DTSTART:YYYYMMDDTHHMMSSZ) while Schedule.next_run, Schedule.dtstart, and Schedule.dtend will be stored in the UTC equivalent (i.e., the scheduler will still do math on "what to run next" based on UTC datetimes). in addition to this change, there is now a new API endpoint, `/api/v2/schedules/preview/`, which takes an rrule and shows the next 10 occurrences in local and UTC time. see: https://github.com/ansible/ansible-tower/issues/823 related: https://github.com/dateutil/dateutil/issues/614 --- awx/api/serializers.py | 115 ++++++------ awx/api/urls/urls.py | 4 + awx/api/views.py | 26 +++ awx/main/models/schedules.py | 95 +++++++++- .../tests/functional/api/test_schedules.py | 126 +++++++++++++ .../tests/functional/models/test_schedule.py | 168 ++++++++++++++++++ docs/schedules.md | 145 +++++++++++++++ 7 files changed, 616 insertions(+), 63 deletions(-) create mode 100644 awx/main/tests/functional/models/test_schedule.py create mode 100644 docs/schedules.md diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c0ad10ce09..0acdd2b34b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -9,7 +9,6 @@ import re import six import urllib from collections import OrderedDict -from dateutil import rrule # Django from django.conf import settings @@ -3873,7 +3872,67 @@ class LabelSerializer(BaseSerializer): return res -class ScheduleSerializer(LaunchConfigurationBaseSerializer): +class SchedulePreviewSerializer(BaseSerializer): + + class Meta: + model = Schedule + fields = ('rrule',) + + # We reject rrules if: + # - DTSTART is not include + # - INTERVAL is not included + # - SECONDLY is used + # - TZID is used + # - BYDAY prefixed with a number (MO is good but not 20MO) + # - BYYEARDAY + # - BYWEEKNO + # - Multiple DTSTART or RRULE elements + # - COUNT > 999 + def validate_rrule(self, value): + rrule_value = value + multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+" + multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+" + by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}" + match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value) + match_multiple_dtstart = re.findall(".*?(DTSTART(;[^:]+)?\:[0-9]+T[0-9]+Z?)", rrule_value) + match_native_dtstart = re.findall(".*?(DTSTART:[0-9]+T[0-9]+) ", rrule_value) + match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value) + if not len(match_multiple_dtstart): + raise serializers.ValidationError(_('Valid DTSTART required in rrule. Value should start with: DTSTART:YYYYMMDDTHHMMSSZ')) + if len(match_native_dtstart): + raise serializers.ValidationError(_('DTSTART cannot be a naive datetime. Specify ;TZINFO= or YYYYMMDDTHHMMSSZZ.')) + if len(match_multiple_dtstart) > 1: + raise serializers.ValidationError(_('Multiple DTSTART is not supported.')) + if not len(match_multiple_rrule): + raise serializers.ValidationError(_('RRULE required in rrule.')) + if len(match_multiple_rrule) > 1: + raise serializers.ValidationError(_('Multiple RRULE is not supported.')) + if 'interval' not in rrule_value.lower(): + raise serializers.ValidationError(_('INTERVAL required in rrule.')) + if 'secondly' in rrule_value.lower(): + raise serializers.ValidationError(_('SECONDLY is not supported.')) + if re.match(multi_by_month_day, rrule_value): + raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.')) + if re.match(multi_by_month, rrule_value): + raise serializers.ValidationError(_('Multiple BYMONTHs not supported.')) + if re.match(by_day_with_numeric_prefix, rrule_value): + raise serializers.ValidationError(_("BYDAY with numeric prefix not supported.")) + if 'byyearday' in rrule_value.lower(): + raise serializers.ValidationError(_("BYYEARDAY not supported.")) + if 'byweekno' in rrule_value.lower(): + raise serializers.ValidationError(_("BYWEEKNO not supported.")) + if match_count: + count_val = match_count.groups()[0].strip().split("=") + if int(count_val[1]) > 999: + raise serializers.ValidationError(_("COUNT > 999 is unsupported.")) + try: + Schedule.rrulestr(rrule_value) + except Exception: + raise serializers.ValidationError(_("rrule parsing failed validation.")) + return value + + +class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSerializer): show_capabilities = ['edit', 'delete'] class Meta: @@ -3900,58 +3959,6 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer): 'Schedule its source project `{}` instead.').format(value.source_project.name))) return value - # We reject rrules if: - # - DTSTART is not include - # - INTERVAL is not included - # - SECONDLY is used - # - TZID is used - # - BYDAY prefixed with a number (MO is good but not 20MO) - # - BYYEARDAY - # - BYWEEKNO - # - Multiple DTSTART or RRULE elements - # - COUNT > 999 - def validate_rrule(self, value): - rrule_value = value - multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+" - multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+" - by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}" - match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value) - match_multiple_dtstart = re.findall(".*?(DTSTART\:[0-9]+T[0-9]+Z)", rrule_value) - match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value) - if not len(match_multiple_dtstart): - raise serializers.ValidationError(_('DTSTART required in rrule. Value should match: DTSTART:YYYYMMDDTHHMMSSZ')) - if len(match_multiple_dtstart) > 1: - raise serializers.ValidationError(_('Multiple DTSTART is not supported.')) - if not len(match_multiple_rrule): - raise serializers.ValidationError(_('RRULE require in rrule.')) - if len(match_multiple_rrule) > 1: - raise serializers.ValidationError(_('Multiple RRULE is not supported.')) - if 'interval' not in rrule_value.lower(): - raise serializers.ValidationError(_('INTERVAL required in rrule.')) - if 'tzid' in rrule_value.lower(): - raise serializers.ValidationError(_('TZID is not supported.')) - if 'secondly' in rrule_value.lower(): - raise serializers.ValidationError(_('SECONDLY is not supported.')) - if re.match(multi_by_month_day, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.')) - if re.match(multi_by_month, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHs not supported.')) - if re.match(by_day_with_numeric_prefix, rrule_value): - raise serializers.ValidationError(_("BYDAY with numeric prefix not supported.")) - if 'byyearday' in rrule_value.lower(): - raise serializers.ValidationError(_("BYYEARDAY not supported.")) - if 'byweekno' in rrule_value.lower(): - raise serializers.ValidationError(_("BYWEEKNO not supported.")) - if match_count: - count_val = match_count.groups()[0].strip().split("=") - if int(count_val[1]) > 999: - raise serializers.ValidationError(_("COUNT > 999 is unsupported.")) - try: - rrule.rrulestr(rrule_value) - except Exception: - raise serializers.ValidationError(_("rrule parsing failed validation.")) - return value - class InstanceSerializer(BaseSerializer): diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 15336f4603..15af2b4dca 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -22,6 +22,8 @@ from awx.api.views import ( JobExtraCredentialsList, JobTemplateCredentialsList, JobTemplateExtraCredentialsList, + SchedulePreview, + ScheduleZoneInfo, ) from .organization import urls as organization_urls @@ -113,6 +115,8 @@ v2_urls = [ url(r'^jobs/(?P[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'), url(r'^job_templates/(?P[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'), url(r'^job_templates/(?P[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'), + url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'), + url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'), ] app_name = 'api' diff --git a/awx/api/views.py b/awx/api/views.py index bd00d05009..0674e74f09 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -53,6 +53,7 @@ import ansiconv # Python Social Auth from social_core.backends.utils import load_backends +import pytz from wsgiref.util import FileWrapper # AWX @@ -608,6 +609,31 @@ class ScheduleDetail(RetrieveUpdateDestroyAPIView): new_in_148 = True +class SchedulePreview(GenericAPIView): + + model = Schedule + view_name = _('Schedule Recurrence Rule Preview') + serializer_class = SchedulePreviewSerializer + new_in_api_v2 = True + + def post(self, request): + serializer = self.get_serializer(data=request.data) + if serializer.is_valid(): + schedule = list(Schedule.rrulestr(serializer.validated_data['rrule']).xafter(now(), count=10)) + return Response({ + 'local': schedule, + 'utc': [s.astimezone(pytz.utc) for s in schedule] + }) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + +class ScheduleZoneInfo(APIView): + + def get(self, request): + from dateutil.zoneinfo import get_zonefile_instance + return Response(sorted(get_zonefile_instance().zones.keys())) + + class LaunchConfigCredentialsBase(SubListAttachDetachAPIView): model = Credential diff --git a/awx/main/models/schedules.py b/awx/main/models/schedules.py index 09fd755d62..19e45c3917 100644 --- a/awx/main/models/schedules.py +++ b/awx/main/models/schedules.py @@ -5,11 +5,12 @@ import re import logging import datetime import dateutil.rrule +from dateutil.tz import gettz # Django from django.db import models from django.db.models.query import QuerySet -from django.utils.timezone import now, make_aware, get_default_timezone +from django.utils.timezone import now, make_aware from django.utils.translation import ugettext_lazy as _ # AWX @@ -19,6 +20,9 @@ from awx.main.models.jobs import LaunchTimeConfig from awx.main.utils import ignore_inventory_computed_fields from awx.main.consumers import emit_channel_notification +import pytz +import six + logger = logging.getLogger('awx.main.models.schedule') @@ -54,6 +58,10 @@ class ScheduleManager(ScheduleFilterMethods, models.Manager): class Schedule(CommonModel, LaunchTimeConfig): + TZID_REGEX = re.compile( + "^(DTSTART;TZID=(?P[^:]+)(?P\:[0-9]+T[0-9]+))(?P .*)$" + ) + class Meta: app_label = 'main' ordering = ['-next_run'] @@ -92,6 +100,74 @@ class Schedule(CommonModel, LaunchTimeConfig): help_text=_("The next time that the scheduled action will run.") ) + @classmethod + def rrulestr(cls, rrule, **kwargs): + """ + Apply our own custom rrule parsing logic. This applies some extensions + and limitations to parsing that are specific to our supported + implementation. Namely: + + * python-dateutil doesn't _natively_ support `DTSTART;TZID=`; this + function parses out the TZID= component and uses it to produce the + `tzinfos` keyword argument to `dateutil.rrule.rrulestr()`. In this + way, we translate: + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1 + + ...into... + + DTSTART:20180601T120000TZID RRULE:FREQ=DAILY;INTERVAL=1 + + ...and we pass a hint about the local timezone to dateutil's parser: + `dateutil.rrule.rrulestr(rrule, { + 'tzinfos': { + 'TZID': dateutil.tz.gettz('America/New_York') + } + })` + + it's possible that we can remove the custom code that performs this + parsing if TZID= gains support in upstream dateutil: + https://github.com/dateutil/dateutil/pull/615 + + * RFC5545 specifies that: if the "DTSTART" property is specified as + a date with local time (in our case, TZID=), then the UNTIL rule part + MUST also be treated as a date with local time. If the "DTSTART" + property is specified as a date with UTC time (with a Z suffix), + then the UNTIL rule part MUST be specified as a date with UTC time. + + this function provides additional parsing to translate RRULES like this: + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T170000 + + ...into this (under the hood): + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T210000Z + """ + source_rrule = rrule + kwargs['tzinfos'] = {} + match = cls.TZID_REGEX.match(rrule) + if match is not None: + rrule = cls.TZID_REGEX.sub("DTSTART\gTZI\g", rrule) + timezone = gettz(match.group('tzid')) + if 'until' in rrule.lower(): + # if DTSTART;TZID= is used, coerce "naive" UNTIL values + # to the proper UTC date + match_until = re.match(".*?(?PUNTIL\=[0-9]+T[0-9]+)(?PZ?)", rrule) + until_date = match_until.group('until').split("=")[1] + if len(match_until.group('utcflag')): + raise ValueError(six.text_type( + _('invalid rrule `{}` includes TZINFO= stanza and UTC-based UNTIL clause').format(source_rrule) + )) # noqa + localized = make_aware( + datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%S"), + timezone + ) + utc = localized.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%SZ') + rrule = rrule.replace(match_until.group('until'), 'UNTIL={}'.format(utc)) + kwargs['tzinfos']['TZI'] = timezone + x = dateutil.rrule.rrulestr(rrule, **kwargs) + return x + def __unicode__(self): return u'%s_t%s_%s_%s' % (self.name, self.unified_job_template.id, self.id, self.next_run) @@ -107,21 +183,22 @@ class Schedule(CommonModel, LaunchTimeConfig): return job_kwargs def update_computed_fields(self): - future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True) + future_rs = Schedule.rrulestr(self.rrule, forceset=True) next_run_actual = future_rs.after(now()) + if next_run_actual is not None: + next_run_actual = next_run_actual.astimezone(pytz.utc) self.next_run = next_run_actual try: - self.dtstart = future_rs[0] + self.dtstart = future_rs[0].astimezone(pytz.utc) except IndexError: self.dtstart = None self.dtend = None - if 'until' in self.rrule.lower(): - match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule) - until_date = match_until.groups()[0].split("=")[1] - self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone()) - if 'count' in self.rrule.lower(): - self.dtend = future_rs[-1] + if 'until' in self.rrule.lower() or 'count' in self.rrule.lower(): + try: + self.dtend = future_rs[-1].astimezone(pytz.utc) + except IndexError: + self.dtend = None emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules')) with ignore_inventory_computed_fields(): self.unified_job_template.update_computed_fields() diff --git a/awx/main/tests/functional/api/test_schedules.py b/awx/main/tests/functional/api/test_schedules.py index a72a59148b..b414241ce1 100644 --- a/awx/main/tests/functional/api/test_schedules.py +++ b/awx/main/tests/functional/api/test_schedules.py @@ -8,6 +8,17 @@ from awx.main.models import JobTemplate RRULE_EXAMPLE = 'DTSTART:20151117T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1' +def get_rrule(tz=None): + parts = ['DTSTART'] + if tz: + parts.append(';TZID={}'.format(tz)) + parts.append(':20300308T050000') + if tz is None: + parts.append('Z') + parts.append(' RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5') + return ''.join(parts) + + @pytest.mark.django_db def test_non_job_extra_vars_prohibited(post, project, admin_user): url = reverse('api:project_schedules_list', kwargs={'pk': project.id}) @@ -32,3 +43,118 @@ def test_valid_survey_answer(post, admin_user, project, inventory, survey_spec_f url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id}) post(url, {'name': 'test sch', 'rrule': RRULE_EXAMPLE, 'extra_data': '{"var1": 54}'}, admin_user, expect=201) + + +@pytest.mark.django_db +@pytest.mark.parametrize('rrule, error', [ + ("", "This field may not be blank"), + ("DTSTART:NONSENSE", "Valid DTSTART required in rrule"), + ("DTSTART:20300308T050000Z DTSTART:20310308T050000", "Multiple DTSTART is not supported"), + ("DTSTART:20300308T050000Z", "RRULE required in rrule"), + ("DTSTART:20300308T050000Z RRULE:NONSENSE", "INTERVAL required in rrule"), + ("DTSTART:20300308T050000Z RRULE:FREQ=SECONDLY;INTERVAL=5;COUNT=6", "SECONDLY is not supported"), + ("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=3,4", "Multiple BYMONTHDAYs not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1,2", "Multiple BYMONTHs not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO", "BYDAY with numeric prefix not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYYEARDAY=100", "BYYEARDAY not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYWEEKNO=20", "BYWEEKNO not supported"), + ("DTSTART:20300308T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000", "COUNT > 999 is unsupported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=REGULARLY;INTERVAL=1", "rrule parsing failed validation"), # noqa + ("DTSTART;TZID=America/New_York:30200308T050000Z RRULE:FREQ=DAILY;INTERVAL=1", "rrule parsing failed validation"), + ("DTSTART:20300308T050000 RRULE:FREQ=DAILY;INTERVAL=1", "DTSTART cannot be a naive datetime"), +]) +def test_invalid_rrules(post, admin_user, project, inventory, rrule, error): + job_template = JobTemplate.objects.create( + name='test-jt', + project=project, + playbook='helloworld.yml', + inventory=inventory + ) + url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id}) + resp = post(url, { + 'name': 'Some Schedule', + 'rrule': rrule, + }, admin_user, expect=400) + assert error in resp.content + + +@pytest.mark.django_db +def test_utc_preview(post, admin_user): + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule()}, admin_user, expect=200) + assert r.data['utc'] == r.data['local'] + assert map(str, r.data['utc']) == [ + '2030-03-08 05:00:00+00:00', + '2030-03-09 05:00:00+00:00', + '2030-03-10 05:00:00+00:00', + '2030-03-11 05:00:00+00:00', + '2030-03-12 05:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_nyc_with_dst(post, admin_user): + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule('America/New_York')}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-03-08 05:00:00-05:00', + '2030-03-09 05:00:00-05:00', + '2030-03-10 05:00:00-04:00', + '2030-03-11 05:00:00-04:00', + '2030-03-12 05:00:00-04:00', + ] + assert map(str, r.data['utc']) == [ + '2030-03-08 10:00:00+00:00', + '2030-03-09 10:00:00+00:00', + '2030-03-10 09:00:00+00:00', + '2030-03-11 09:00:00+00:00', + '2030-03-12 09:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_phoenix_without_dst(post, admin_user): + # The state of Arizona (aside from a few Native American territories) does + # not observe DST + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule('America/Phoenix')}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-03-08 05:00:00-07:00', + '2030-03-09 05:00:00-07:00', + '2030-03-10 05:00:00-07:00', + '2030-03-11 05:00:00-07:00', + '2030-03-12 05:00:00-07:00', + ] + assert map(str, r.data['utc']) == [ + '2030-03-08 12:00:00+00:00', + '2030-03-09 12:00:00+00:00', + '2030-03-10 12:00:00+00:00', + '2030-03-11 12:00:00+00:00', + '2030-03-12 12:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_interval_by_local_day(post, admin_user): + url = reverse('api:schedule_rrule') + rrule = 'DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4' + r = post(url, {'rrule': rrule}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-02-02 21:00:00-05:00', + '2030-03-02 21:00:00-05:00', + '2030-04-06 21:00:00-04:00', + '2030-05-04 21:00:00-04:00', + ] + + assert map(str, r.data['utc']) == [ + '2030-02-03 02:00:00+00:00', + '2030-03-03 02:00:00+00:00', + '2030-04-07 01:00:00+00:00', + '2030-05-05 01:00:00+00:00', + ] diff --git a/awx/main/tests/functional/models/test_schedule.py b/awx/main/tests/functional/models/test_schedule.py new file mode 100644 index 0000000000..8e1df57ce2 --- /dev/null +++ b/awx/main/tests/functional/models/test_schedule.py @@ -0,0 +1,168 @@ +from awx.main.models import JobTemplate, Schedule +import pytest + + +@pytest.fixture +def job_template(inventory, project): + # need related resources set for these tests + return JobTemplate.objects.create( + name='test-job_template', + inventory=inventory, + project=project + ) + + +@pytest.mark.django_db +def test_repeats_forever(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == '2030-01-12 21:00:00+00:00' + assert s.dtend is None + + +@pytest.mark.django_db +def test_no_recurrence_utc(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-12 21:00:00+00:00' + + +@pytest.mark.django_db +def test_no_recurrence_est(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-13 02:00:00+00:00' + + +@pytest.mark.django_db +def test_next_run_utc(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == '2030-02-02 21:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2030-05-04 21:00:00+00:00' + + +@pytest.mark.django_db +def test_next_run_est(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4', + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2030-02-03 02:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + + # March 10, 2030 is when DST takes effect in NYC + assert str(s.dtend) == '2030-05-05 01:00:00+00:00' + + +@pytest.mark.django_db +def test_year_boundary(job_template): + rrule = 'DTSTART;TZID=America/New_York:20301231T230000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=12;BYMONTHDAY=31;COUNT=4' # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2031-01-01 04:00:00+00:00' # UTC = +5 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2034-01-01 04:00:00+00:00' # UTC = +5 EST + + +@pytest.mark.django_db +def test_leap_year_day(job_template): + rrule = 'DTSTART;TZID=America/New_York:20320229T050000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=02;BYMONTHDAY=29;COUNT=2' # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2032-02-29 10:00:00+00:00' # UTC = +5 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2036-02-29 10:00:00+00:00' # UTC = +5 EST + + +@pytest.mark.django_db +@pytest.mark.parametrize('until, dtend', [ + ['20180602T170000Z', '2018-06-02 12:00:00+00:00'], + ['20180602T000000Z', '2018-06-01 12:00:00+00:00'], +]) +def test_utc_until(job_template, until, dtend): + rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until) + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2018-06-01 12:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == dtend + + +@pytest.mark.django_db +@pytest.mark.parametrize('until, dtend', [ + ['20180602T170000', '2018-06-02 16:00:00+00:00'], + ['20180602T000000', '2018-06-01 16:00:00+00:00'], +]) +def test_tzinfo_until(job_template, until, dtend): + rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until) # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2018-06-01 16:00:00+00:00' # UTC = +4 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == dtend + + +@pytest.mark.django_db +def test_mismatched_until_timezone(job_template): + rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T000000' + 'Z' # noqa the Z isn't allowed, because we have a TZID=America/New_York + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + with pytest.raises(ValueError): + s.save() + + +@pytest.mark.django_db +def test_utc_until_in_the_past(job_template): + rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20150101T100000Z' + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert s.next_run is s.dtstart is s.dtend is None diff --git a/docs/schedules.md b/docs/schedules.md new file mode 100644 index 0000000000..be826bf388 --- /dev/null +++ b/docs/schedules.md @@ -0,0 +1,145 @@ +Scheduled Jobs +============== + +awx allows jobs to run on a schedule (with optional recurrence rules) via +an `HTTP POST` to a variety of API endpoints: + + HTTP POST + + https://tower-host.example.org/api/v2/job_templates/N/schedules/ + https://tower-host.example.org/api/v2/projects/N/schedules/ + https://tower-host.example.org/api/v2/inventory_sources/N/schedules/ + https://tower-host.example.org/api/v2/system_jobs/N/schedules/ + https://tower-host.example.org/api/v2/workflow_job_templates/N/schedules/ + + { + 'name': 'My Schedule Name', + 'rrule': 'DTSTART:20300115T120000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7' + 'extra_data': {} + } + +...where `rrule` is a valid +[RFC5545](https://www.rfc-editor.org/rfc/rfc5545.txt) RRULE string. The +specific example above would run a job every day - for seven consecutive days - starting +on January 15th, 2030 at noon (UTC). + +Specifying Timezones +==================== +`DTSTART` values provided to awx _must_ provide timezone information (they may +not be naive dates). + +For UTC dates, `DTSTART` values should be denoted with the `Z` suffix: + + DTSTART:20300115T120000Z + +Local timezones can be specified using the `TZID=` parameter: + + DTSTART;TZID=America/New_York:20300115T120000 + +A list of _valid_ zone identifiers (which can vary by system) can be found at: + + HTTP GET /api/v2/schedules/zoneinfo/ + + [ + "Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + ... + ] + + +UNTIL and Timezones +=================== +RFC5545 specifies that: + +> Furthermore, if the "DTSTART" property is specified as a date with local +> time, then the UNTIL rule part MUST also be specified as a date with local +> time. If the "DTSTART" property is specified as a date with UTC time or +> a date with local time and time zone reference, then the UNTIL rule part +> MUST be specified as a date with UTC time. + +Given this, this RRULE: + + `DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000Z` + +...will be interpretted as "Starting on June 1st, 2018 at noon UTC, repeat +daily, ending on June 6th, 2018 at 5PM UTC". + +This RRULE: + + `DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000` + +...will be interpretted as "Starting on June 1st, 2018 at noon EDT, repeat +daily, ending on June 6th, 2018 at 5PM EDT". + + +Previewing Schedules +==================== +awx provides an endpoint for previewing the future dates and times for +a specified `RRULE`. A list of the next _ten_ occurrences will be returned in +local and UTC time: + + POST https://tower-host.example.org/api/v2/schedules/preview/ + { + 'rrule': 'DTSTART;TZID=America/New_York:20300115T120000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7' + } + + Content-Type: application/json + { + "local": [ + "2030-01-15T12:00:00-05:00", + "2030-01-16T12:00:00-05:00", + "2030-01-17T12:00:00-05:00", + "2030-01-18T12:00:00-05:00", + "2030-01-19T12:00:00-05:00", + "2030-01-20T12:00:00-05:00", + "2030-01-21T12:00:00-05:00" + ], + "utc": [ + "2030-01-15T17:00:00Z", + "2030-01-16T17:00:00Z", + "2030-01-17T17:00:00Z", + "2030-01-18T17:00:00Z", + "2030-01-19T17:00:00Z", + "2030-01-20T17:00:00Z", + "2030-01-21T17:00:00Z" + ] + } + + +RRULE Limitations +================= + +The following aspects of `RFC5545` are _not_ supported by awx schedules: + +* Strings with more than a single `DTSTART:` component +* Strings with more than a single `RRULE` component +* The use of `FREQ=SECONDLY` in an `RRULE` +* The use of more than a single `FREQ=BYMONTHDAY` component in an `RRULE` +* The use of more than a single `FREQ=BYMONTHS` component in an `RRULE` +* The use of `FREQ=BYYEARDAY` in an `RRULE` +* The use of `FREQ=BYWEEKNO` in an `RRULE` +* The use of `FREQ=BYWEEKNO` in an `RRULE` +* The use of `COUNT=` in an `RRULE` with a value over 999 + + +Implementation Details +====================== + +Any time an `awx.model.Schedule` is saved with a valid `rrule` value, the +`dateutil` library is used to burst out a list of all occurrences. From here, +the following dates are saved in the database: + +* `main_schedule.rrule` - the original `RRULE` string provided by the user +* `main_schedule.dtstart` - the _first_ datetime in the list of all occurrences (coerced to UTC) +* `main_schedule.dtend` - the _last_ datetime in the list of all occurrences (coerced to UTC) +* `main_schedule.next_run` - the _next_ datetime in list after `utcnow()` (coerced to UTC) + +awx makes use of [Celery Periodic Tasks +(celerybeat)](http://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html) +to run a periodic task that discovers new jobs that need to run at a regular +interval (by default, every 30 seconds). When this task starts, it queries the +database for Schedules where `Schedule.next_run` is between +`scheduler_last_runtime()` and `utcnow()`. For each of these, a new job is +launched, and `Schedule.next_run` is changed to the next chronological datetime +in the list of all occurences.