diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c0ad10ce09..0acdd2b34b 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -9,7 +9,6 @@ import re import six import urllib from collections import OrderedDict -from dateutil import rrule # Django from django.conf import settings @@ -3873,7 +3872,67 @@ class LabelSerializer(BaseSerializer): return res -class ScheduleSerializer(LaunchConfigurationBaseSerializer): +class SchedulePreviewSerializer(BaseSerializer): + + class Meta: + model = Schedule + fields = ('rrule',) + + # We reject rrules if: + # - DTSTART is not include + # - INTERVAL is not included + # - SECONDLY is used + # - TZID is used + # - BYDAY prefixed with a number (MO is good but not 20MO) + # - BYYEARDAY + # - BYWEEKNO + # - Multiple DTSTART or RRULE elements + # - COUNT > 999 + def validate_rrule(self, value): + rrule_value = value + multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+" + multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+" + by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}" + match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value) + match_multiple_dtstart = re.findall(".*?(DTSTART(;[^:]+)?\:[0-9]+T[0-9]+Z?)", rrule_value) + match_native_dtstart = re.findall(".*?(DTSTART:[0-9]+T[0-9]+) ", rrule_value) + match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value) + if not len(match_multiple_dtstart): + raise serializers.ValidationError(_('Valid DTSTART required in rrule. Value should start with: DTSTART:YYYYMMDDTHHMMSSZ')) + if len(match_native_dtstart): + raise serializers.ValidationError(_('DTSTART cannot be a naive datetime. Specify ;TZINFO= or YYYYMMDDTHHMMSSZZ.')) + if len(match_multiple_dtstart) > 1: + raise serializers.ValidationError(_('Multiple DTSTART is not supported.')) + if not len(match_multiple_rrule): + raise serializers.ValidationError(_('RRULE required in rrule.')) + if len(match_multiple_rrule) > 1: + raise serializers.ValidationError(_('Multiple RRULE is not supported.')) + if 'interval' not in rrule_value.lower(): + raise serializers.ValidationError(_('INTERVAL required in rrule.')) + if 'secondly' in rrule_value.lower(): + raise serializers.ValidationError(_('SECONDLY is not supported.')) + if re.match(multi_by_month_day, rrule_value): + raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.')) + if re.match(multi_by_month, rrule_value): + raise serializers.ValidationError(_('Multiple BYMONTHs not supported.')) + if re.match(by_day_with_numeric_prefix, rrule_value): + raise serializers.ValidationError(_("BYDAY with numeric prefix not supported.")) + if 'byyearday' in rrule_value.lower(): + raise serializers.ValidationError(_("BYYEARDAY not supported.")) + if 'byweekno' in rrule_value.lower(): + raise serializers.ValidationError(_("BYWEEKNO not supported.")) + if match_count: + count_val = match_count.groups()[0].strip().split("=") + if int(count_val[1]) > 999: + raise serializers.ValidationError(_("COUNT > 999 is unsupported.")) + try: + Schedule.rrulestr(rrule_value) + except Exception: + raise serializers.ValidationError(_("rrule parsing failed validation.")) + return value + + +class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSerializer): show_capabilities = ['edit', 'delete'] class Meta: @@ -3900,58 +3959,6 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer): 'Schedule its source project `{}` instead.').format(value.source_project.name))) return value - # We reject rrules if: - # - DTSTART is not include - # - INTERVAL is not included - # - SECONDLY is used - # - TZID is used - # - BYDAY prefixed with a number (MO is good but not 20MO) - # - BYYEARDAY - # - BYWEEKNO - # - Multiple DTSTART or RRULE elements - # - COUNT > 999 - def validate_rrule(self, value): - rrule_value = value - multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+" - multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+" - by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}" - match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value) - match_multiple_dtstart = re.findall(".*?(DTSTART\:[0-9]+T[0-9]+Z)", rrule_value) - match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value) - if not len(match_multiple_dtstart): - raise serializers.ValidationError(_('DTSTART required in rrule. Value should match: DTSTART:YYYYMMDDTHHMMSSZ')) - if len(match_multiple_dtstart) > 1: - raise serializers.ValidationError(_('Multiple DTSTART is not supported.')) - if not len(match_multiple_rrule): - raise serializers.ValidationError(_('RRULE require in rrule.')) - if len(match_multiple_rrule) > 1: - raise serializers.ValidationError(_('Multiple RRULE is not supported.')) - if 'interval' not in rrule_value.lower(): - raise serializers.ValidationError(_('INTERVAL required in rrule.')) - if 'tzid' in rrule_value.lower(): - raise serializers.ValidationError(_('TZID is not supported.')) - if 'secondly' in rrule_value.lower(): - raise serializers.ValidationError(_('SECONDLY is not supported.')) - if re.match(multi_by_month_day, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.')) - if re.match(multi_by_month, rrule_value): - raise serializers.ValidationError(_('Multiple BYMONTHs not supported.')) - if re.match(by_day_with_numeric_prefix, rrule_value): - raise serializers.ValidationError(_("BYDAY with numeric prefix not supported.")) - if 'byyearday' in rrule_value.lower(): - raise serializers.ValidationError(_("BYYEARDAY not supported.")) - if 'byweekno' in rrule_value.lower(): - raise serializers.ValidationError(_("BYWEEKNO not supported.")) - if match_count: - count_val = match_count.groups()[0].strip().split("=") - if int(count_val[1]) > 999: - raise serializers.ValidationError(_("COUNT > 999 is unsupported.")) - try: - rrule.rrulestr(rrule_value) - except Exception: - raise serializers.ValidationError(_("rrule parsing failed validation.")) - return value - class InstanceSerializer(BaseSerializer): diff --git a/awx/api/urls/urls.py b/awx/api/urls/urls.py index 15336f4603..15af2b4dca 100644 --- a/awx/api/urls/urls.py +++ b/awx/api/urls/urls.py @@ -22,6 +22,8 @@ from awx.api.views import ( JobExtraCredentialsList, JobTemplateCredentialsList, JobTemplateExtraCredentialsList, + SchedulePreview, + ScheduleZoneInfo, ) from .organization import urls as organization_urls @@ -113,6 +115,8 @@ v2_urls = [ url(r'^jobs/(?P[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'), url(r'^job_templates/(?P[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'), url(r'^job_templates/(?P[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'), + url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'), + url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'), ] app_name = 'api' diff --git a/awx/api/views.py b/awx/api/views.py index bd00d05009..0674e74f09 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -53,6 +53,7 @@ import ansiconv # Python Social Auth from social_core.backends.utils import load_backends +import pytz from wsgiref.util import FileWrapper # AWX @@ -608,6 +609,31 @@ class ScheduleDetail(RetrieveUpdateDestroyAPIView): new_in_148 = True +class SchedulePreview(GenericAPIView): + + model = Schedule + view_name = _('Schedule Recurrence Rule Preview') + serializer_class = SchedulePreviewSerializer + new_in_api_v2 = True + + def post(self, request): + serializer = self.get_serializer(data=request.data) + if serializer.is_valid(): + schedule = list(Schedule.rrulestr(serializer.validated_data['rrule']).xafter(now(), count=10)) + return Response({ + 'local': schedule, + 'utc': [s.astimezone(pytz.utc) for s in schedule] + }) + return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST) + + +class ScheduleZoneInfo(APIView): + + def get(self, request): + from dateutil.zoneinfo import get_zonefile_instance + return Response(sorted(get_zonefile_instance().zones.keys())) + + class LaunchConfigCredentialsBase(SubListAttachDetachAPIView): model = Credential diff --git a/awx/main/models/schedules.py b/awx/main/models/schedules.py index 09fd755d62..19e45c3917 100644 --- a/awx/main/models/schedules.py +++ b/awx/main/models/schedules.py @@ -5,11 +5,12 @@ import re import logging import datetime import dateutil.rrule +from dateutil.tz import gettz # Django from django.db import models from django.db.models.query import QuerySet -from django.utils.timezone import now, make_aware, get_default_timezone +from django.utils.timezone import now, make_aware from django.utils.translation import ugettext_lazy as _ # AWX @@ -19,6 +20,9 @@ from awx.main.models.jobs import LaunchTimeConfig from awx.main.utils import ignore_inventory_computed_fields from awx.main.consumers import emit_channel_notification +import pytz +import six + logger = logging.getLogger('awx.main.models.schedule') @@ -54,6 +58,10 @@ class ScheduleManager(ScheduleFilterMethods, models.Manager): class Schedule(CommonModel, LaunchTimeConfig): + TZID_REGEX = re.compile( + "^(DTSTART;TZID=(?P[^:]+)(?P\:[0-9]+T[0-9]+))(?P .*)$" + ) + class Meta: app_label = 'main' ordering = ['-next_run'] @@ -92,6 +100,74 @@ class Schedule(CommonModel, LaunchTimeConfig): help_text=_("The next time that the scheduled action will run.") ) + @classmethod + def rrulestr(cls, rrule, **kwargs): + """ + Apply our own custom rrule parsing logic. This applies some extensions + and limitations to parsing that are specific to our supported + implementation. Namely: + + * python-dateutil doesn't _natively_ support `DTSTART;TZID=`; this + function parses out the TZID= component and uses it to produce the + `tzinfos` keyword argument to `dateutil.rrule.rrulestr()`. In this + way, we translate: + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1 + + ...into... + + DTSTART:20180601T120000TZID RRULE:FREQ=DAILY;INTERVAL=1 + + ...and we pass a hint about the local timezone to dateutil's parser: + `dateutil.rrule.rrulestr(rrule, { + 'tzinfos': { + 'TZID': dateutil.tz.gettz('America/New_York') + } + })` + + it's possible that we can remove the custom code that performs this + parsing if TZID= gains support in upstream dateutil: + https://github.com/dateutil/dateutil/pull/615 + + * RFC5545 specifies that: if the "DTSTART" property is specified as + a date with local time (in our case, TZID=), then the UNTIL rule part + MUST also be treated as a date with local time. If the "DTSTART" + property is specified as a date with UTC time (with a Z suffix), + then the UNTIL rule part MUST be specified as a date with UTC time. + + this function provides additional parsing to translate RRULES like this: + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T170000 + + ...into this (under the hood): + + DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T210000Z + """ + source_rrule = rrule + kwargs['tzinfos'] = {} + match = cls.TZID_REGEX.match(rrule) + if match is not None: + rrule = cls.TZID_REGEX.sub("DTSTART\gTZI\g", rrule) + timezone = gettz(match.group('tzid')) + if 'until' in rrule.lower(): + # if DTSTART;TZID= is used, coerce "naive" UNTIL values + # to the proper UTC date + match_until = re.match(".*?(?PUNTIL\=[0-9]+T[0-9]+)(?PZ?)", rrule) + until_date = match_until.group('until').split("=")[1] + if len(match_until.group('utcflag')): + raise ValueError(six.text_type( + _('invalid rrule `{}` includes TZINFO= stanza and UTC-based UNTIL clause').format(source_rrule) + )) # noqa + localized = make_aware( + datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%S"), + timezone + ) + utc = localized.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%SZ') + rrule = rrule.replace(match_until.group('until'), 'UNTIL={}'.format(utc)) + kwargs['tzinfos']['TZI'] = timezone + x = dateutil.rrule.rrulestr(rrule, **kwargs) + return x + def __unicode__(self): return u'%s_t%s_%s_%s' % (self.name, self.unified_job_template.id, self.id, self.next_run) @@ -107,21 +183,22 @@ class Schedule(CommonModel, LaunchTimeConfig): return job_kwargs def update_computed_fields(self): - future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True) + future_rs = Schedule.rrulestr(self.rrule, forceset=True) next_run_actual = future_rs.after(now()) + if next_run_actual is not None: + next_run_actual = next_run_actual.astimezone(pytz.utc) self.next_run = next_run_actual try: - self.dtstart = future_rs[0] + self.dtstart = future_rs[0].astimezone(pytz.utc) except IndexError: self.dtstart = None self.dtend = None - if 'until' in self.rrule.lower(): - match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule) - until_date = match_until.groups()[0].split("=")[1] - self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone()) - if 'count' in self.rrule.lower(): - self.dtend = future_rs[-1] + if 'until' in self.rrule.lower() or 'count' in self.rrule.lower(): + try: + self.dtend = future_rs[-1].astimezone(pytz.utc) + except IndexError: + self.dtend = None emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules')) with ignore_inventory_computed_fields(): self.unified_job_template.update_computed_fields() diff --git a/awx/main/tests/functional/api/test_schedules.py b/awx/main/tests/functional/api/test_schedules.py index a72a59148b..b414241ce1 100644 --- a/awx/main/tests/functional/api/test_schedules.py +++ b/awx/main/tests/functional/api/test_schedules.py @@ -8,6 +8,17 @@ from awx.main.models import JobTemplate RRULE_EXAMPLE = 'DTSTART:20151117T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1' +def get_rrule(tz=None): + parts = ['DTSTART'] + if tz: + parts.append(';TZID={}'.format(tz)) + parts.append(':20300308T050000') + if tz is None: + parts.append('Z') + parts.append(' RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5') + return ''.join(parts) + + @pytest.mark.django_db def test_non_job_extra_vars_prohibited(post, project, admin_user): url = reverse('api:project_schedules_list', kwargs={'pk': project.id}) @@ -32,3 +43,118 @@ def test_valid_survey_answer(post, admin_user, project, inventory, survey_spec_f url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id}) post(url, {'name': 'test sch', 'rrule': RRULE_EXAMPLE, 'extra_data': '{"var1": 54}'}, admin_user, expect=201) + + +@pytest.mark.django_db +@pytest.mark.parametrize('rrule, error', [ + ("", "This field may not be blank"), + ("DTSTART:NONSENSE", "Valid DTSTART required in rrule"), + ("DTSTART:20300308T050000Z DTSTART:20310308T050000", "Multiple DTSTART is not supported"), + ("DTSTART:20300308T050000Z", "RRULE required in rrule"), + ("DTSTART:20300308T050000Z RRULE:NONSENSE", "INTERVAL required in rrule"), + ("DTSTART:20300308T050000Z RRULE:FREQ=SECONDLY;INTERVAL=5;COUNT=6", "SECONDLY is not supported"), + ("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=3,4", "Multiple BYMONTHDAYs not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1,2", "Multiple BYMONTHs not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO", "BYDAY with numeric prefix not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYYEARDAY=100", "BYYEARDAY not supported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYWEEKNO=20", "BYWEEKNO not supported"), + ("DTSTART:20300308T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000", "COUNT > 999 is unsupported"), # noqa + ("DTSTART:20300308T050000Z RRULE:FREQ=REGULARLY;INTERVAL=1", "rrule parsing failed validation"), # noqa + ("DTSTART;TZID=America/New_York:30200308T050000Z RRULE:FREQ=DAILY;INTERVAL=1", "rrule parsing failed validation"), + ("DTSTART:20300308T050000 RRULE:FREQ=DAILY;INTERVAL=1", "DTSTART cannot be a naive datetime"), +]) +def test_invalid_rrules(post, admin_user, project, inventory, rrule, error): + job_template = JobTemplate.objects.create( + name='test-jt', + project=project, + playbook='helloworld.yml', + inventory=inventory + ) + url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id}) + resp = post(url, { + 'name': 'Some Schedule', + 'rrule': rrule, + }, admin_user, expect=400) + assert error in resp.content + + +@pytest.mark.django_db +def test_utc_preview(post, admin_user): + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule()}, admin_user, expect=200) + assert r.data['utc'] == r.data['local'] + assert map(str, r.data['utc']) == [ + '2030-03-08 05:00:00+00:00', + '2030-03-09 05:00:00+00:00', + '2030-03-10 05:00:00+00:00', + '2030-03-11 05:00:00+00:00', + '2030-03-12 05:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_nyc_with_dst(post, admin_user): + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule('America/New_York')}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-03-08 05:00:00-05:00', + '2030-03-09 05:00:00-05:00', + '2030-03-10 05:00:00-04:00', + '2030-03-11 05:00:00-04:00', + '2030-03-12 05:00:00-04:00', + ] + assert map(str, r.data['utc']) == [ + '2030-03-08 10:00:00+00:00', + '2030-03-09 10:00:00+00:00', + '2030-03-10 09:00:00+00:00', + '2030-03-11 09:00:00+00:00', + '2030-03-12 09:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_phoenix_without_dst(post, admin_user): + # The state of Arizona (aside from a few Native American territories) does + # not observe DST + url = reverse('api:schedule_rrule') + r = post(url, {'rrule': get_rrule('America/Phoenix')}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-03-08 05:00:00-07:00', + '2030-03-09 05:00:00-07:00', + '2030-03-10 05:00:00-07:00', + '2030-03-11 05:00:00-07:00', + '2030-03-12 05:00:00-07:00', + ] + assert map(str, r.data['utc']) == [ + '2030-03-08 12:00:00+00:00', + '2030-03-09 12:00:00+00:00', + '2030-03-10 12:00:00+00:00', + '2030-03-11 12:00:00+00:00', + '2030-03-12 12:00:00+00:00', + ] + + +@pytest.mark.django_db +def test_interval_by_local_day(post, admin_user): + url = reverse('api:schedule_rrule') + rrule = 'DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4' + r = post(url, {'rrule': rrule}, admin_user, expect=200) + + # March 10, 2030 is when DST takes effect in NYC + assert map(str, r.data['local']) == [ + '2030-02-02 21:00:00-05:00', + '2030-03-02 21:00:00-05:00', + '2030-04-06 21:00:00-04:00', + '2030-05-04 21:00:00-04:00', + ] + + assert map(str, r.data['utc']) == [ + '2030-02-03 02:00:00+00:00', + '2030-03-03 02:00:00+00:00', + '2030-04-07 01:00:00+00:00', + '2030-05-05 01:00:00+00:00', + ] diff --git a/awx/main/tests/functional/models/test_schedule.py b/awx/main/tests/functional/models/test_schedule.py new file mode 100644 index 0000000000..8e1df57ce2 --- /dev/null +++ b/awx/main/tests/functional/models/test_schedule.py @@ -0,0 +1,168 @@ +from awx.main.models import JobTemplate, Schedule +import pytest + + +@pytest.fixture +def job_template(inventory, project): + # need related resources set for these tests + return JobTemplate.objects.create( + name='test-job_template', + inventory=inventory, + project=project + ) + + +@pytest.mark.django_db +def test_repeats_forever(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == '2030-01-12 21:00:00+00:00' + assert s.dtend is None + + +@pytest.mark.django_db +def test_no_recurrence_utc(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-12 21:00:00+00:00' + + +@pytest.mark.django_db +def test_no_recurrence_est(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-13 02:00:00+00:00' + + +@pytest.mark.django_db +def test_next_run_utc(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART:20300112T210000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4', + unified_job_template=job_template + ) + s.save() + assert str(s.next_run) == '2030-02-02 21:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2030-05-04 21:00:00+00:00' + + +@pytest.mark.django_db +def test_next_run_est(job_template): + s = Schedule( + name='Some Schedule', + rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4', + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2030-02-03 02:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + + # March 10, 2030 is when DST takes effect in NYC + assert str(s.dtend) == '2030-05-05 01:00:00+00:00' + + +@pytest.mark.django_db +def test_year_boundary(job_template): + rrule = 'DTSTART;TZID=America/New_York:20301231T230000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=12;BYMONTHDAY=31;COUNT=4' # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2031-01-01 04:00:00+00:00' # UTC = +5 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2034-01-01 04:00:00+00:00' # UTC = +5 EST + + +@pytest.mark.django_db +def test_leap_year_day(job_template): + rrule = 'DTSTART;TZID=America/New_York:20320229T050000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=02;BYMONTHDAY=29;COUNT=2' # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2032-02-29 10:00:00+00:00' # UTC = +5 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == '2036-02-29 10:00:00+00:00' # UTC = +5 EST + + +@pytest.mark.django_db +@pytest.mark.parametrize('until, dtend', [ + ['20180602T170000Z', '2018-06-02 12:00:00+00:00'], + ['20180602T000000Z', '2018-06-01 12:00:00+00:00'], +]) +def test_utc_until(job_template, until, dtend): + rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until) + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2018-06-01 12:00:00+00:00' + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == dtend + + +@pytest.mark.django_db +@pytest.mark.parametrize('until, dtend', [ + ['20180602T170000', '2018-06-02 16:00:00+00:00'], + ['20180602T000000', '2018-06-01 16:00:00+00:00'], +]) +def test_tzinfo_until(job_template, until, dtend): + rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until) # noqa + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert str(s.next_run) == '2018-06-01 16:00:00+00:00' # UTC = +4 EST + assert str(s.next_run) == str(s.dtstart) + assert str(s.dtend) == dtend + + +@pytest.mark.django_db +def test_mismatched_until_timezone(job_template): + rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T000000' + 'Z' # noqa the Z isn't allowed, because we have a TZID=America/New_York + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + with pytest.raises(ValueError): + s.save() + + +@pytest.mark.django_db +def test_utc_until_in_the_past(job_template): + rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20150101T100000Z' + s = Schedule( + name='Some Schedule', + rrule=rrule, + unified_job_template=job_template + ) + s.save() + + assert s.next_run is s.dtstart is s.dtend is None diff --git a/docs/schedules.md b/docs/schedules.md new file mode 100644 index 0000000000..be826bf388 --- /dev/null +++ b/docs/schedules.md @@ -0,0 +1,145 @@ +Scheduled Jobs +============== + +awx allows jobs to run on a schedule (with optional recurrence rules) via +an `HTTP POST` to a variety of API endpoints: + + HTTP POST + + https://tower-host.example.org/api/v2/job_templates/N/schedules/ + https://tower-host.example.org/api/v2/projects/N/schedules/ + https://tower-host.example.org/api/v2/inventory_sources/N/schedules/ + https://tower-host.example.org/api/v2/system_jobs/N/schedules/ + https://tower-host.example.org/api/v2/workflow_job_templates/N/schedules/ + + { + 'name': 'My Schedule Name', + 'rrule': 'DTSTART:20300115T120000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7' + 'extra_data': {} + } + +...where `rrule` is a valid +[RFC5545](https://www.rfc-editor.org/rfc/rfc5545.txt) RRULE string. The +specific example above would run a job every day - for seven consecutive days - starting +on January 15th, 2030 at noon (UTC). + +Specifying Timezones +==================== +`DTSTART` values provided to awx _must_ provide timezone information (they may +not be naive dates). + +For UTC dates, `DTSTART` values should be denoted with the `Z` suffix: + + DTSTART:20300115T120000Z + +Local timezones can be specified using the `TZID=` parameter: + + DTSTART;TZID=America/New_York:20300115T120000 + +A list of _valid_ zone identifiers (which can vary by system) can be found at: + + HTTP GET /api/v2/schedules/zoneinfo/ + + [ + "Africa/Abidjan", + "Africa/Accra", + "Africa/Addis_Ababa", + ... + ] + + +UNTIL and Timezones +=================== +RFC5545 specifies that: + +> Furthermore, if the "DTSTART" property is specified as a date with local +> time, then the UNTIL rule part MUST also be specified as a date with local +> time. If the "DTSTART" property is specified as a date with UTC time or +> a date with local time and time zone reference, then the UNTIL rule part +> MUST be specified as a date with UTC time. + +Given this, this RRULE: + + `DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000Z` + +...will be interpretted as "Starting on June 1st, 2018 at noon UTC, repeat +daily, ending on June 6th, 2018 at 5PM UTC". + +This RRULE: + + `DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000` + +...will be interpretted as "Starting on June 1st, 2018 at noon EDT, repeat +daily, ending on June 6th, 2018 at 5PM EDT". + + +Previewing Schedules +==================== +awx provides an endpoint for previewing the future dates and times for +a specified `RRULE`. A list of the next _ten_ occurrences will be returned in +local and UTC time: + + POST https://tower-host.example.org/api/v2/schedules/preview/ + { + 'rrule': 'DTSTART;TZID=America/New_York:20300115T120000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7' + } + + Content-Type: application/json + { + "local": [ + "2030-01-15T12:00:00-05:00", + "2030-01-16T12:00:00-05:00", + "2030-01-17T12:00:00-05:00", + "2030-01-18T12:00:00-05:00", + "2030-01-19T12:00:00-05:00", + "2030-01-20T12:00:00-05:00", + "2030-01-21T12:00:00-05:00" + ], + "utc": [ + "2030-01-15T17:00:00Z", + "2030-01-16T17:00:00Z", + "2030-01-17T17:00:00Z", + "2030-01-18T17:00:00Z", + "2030-01-19T17:00:00Z", + "2030-01-20T17:00:00Z", + "2030-01-21T17:00:00Z" + ] + } + + +RRULE Limitations +================= + +The following aspects of `RFC5545` are _not_ supported by awx schedules: + +* Strings with more than a single `DTSTART:` component +* Strings with more than a single `RRULE` component +* The use of `FREQ=SECONDLY` in an `RRULE` +* The use of more than a single `FREQ=BYMONTHDAY` component in an `RRULE` +* The use of more than a single `FREQ=BYMONTHS` component in an `RRULE` +* The use of `FREQ=BYYEARDAY` in an `RRULE` +* The use of `FREQ=BYWEEKNO` in an `RRULE` +* The use of `FREQ=BYWEEKNO` in an `RRULE` +* The use of `COUNT=` in an `RRULE` with a value over 999 + + +Implementation Details +====================== + +Any time an `awx.model.Schedule` is saved with a valid `rrule` value, the +`dateutil` library is used to burst out a list of all occurrences. From here, +the following dates are saved in the database: + +* `main_schedule.rrule` - the original `RRULE` string provided by the user +* `main_schedule.dtstart` - the _first_ datetime in the list of all occurrences (coerced to UTC) +* `main_schedule.dtend` - the _last_ datetime in the list of all occurrences (coerced to UTC) +* `main_schedule.next_run` - the _next_ datetime in list after `utcnow()` (coerced to UTC) + +awx makes use of [Celery Periodic Tasks +(celerybeat)](http://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html) +to run a periodic task that discovers new jobs that need to run at a regular +interval (by default, every 30 seconds). When this task starts, it queries the +database for Schedules where `Schedule.next_run` is between +`scheduler_last_runtime()` and `utcnow()`. For each of these, a new job is +launched, and `Schedule.next_run` is changed to the next chronological datetime +in the list of all occurences.