External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-25 12:55:04 -02:30
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1024 from ryanpetrello/fix-710-schedule-timezone

Browse Source 
support TZID= in schedule rrules
This commit is contained in:
Ryan Petrello
2018-01-25 10:14:57 -05:00
committed by GitHub
parent fefa4a8bf4 9857c8272e
commit 50290a9063
7 changed files with 773 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
awx/api/serializers.py
View File

@@ -9,7 +9,6 @@ import re
import six
import urllib
from collections import OrderedDict
from dateutil import rrule
# Django
from django.conf import settings
@@ -3873,7 +3872,67 @@ class LabelSerializer(BaseSerializer):
return res
class ScheduleSerializer(LaunchConfigurationBaseSerializer):
class SchedulePreviewSerializer(BaseSerializer):
class Meta:
model = Schedule
fields = ('rrule',)
# We reject rrules if:
# - DTSTART is not include
# - INTERVAL is not included
# - SECONDLY is used
# - TZID is used
# - BYDAY prefixed with a number (MO is good but not 20MO)
# - BYYEARDAY
# - BYWEEKNO
# - Multiple DTSTART or RRULE elements
# - COUNT > 999
def validate_rrule(self, value):
rrule_value = value
multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+"
multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+"
by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}"
match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value)
match_multiple_dtstart = re.findall(".*?(DTSTART(;[^:]+)?\:[0-9]+T[0-9]+Z?)", rrule_value)
match_native_dtstart = re.findall(".*?(DTSTART:[0-9]+T[0-9]+) ", rrule_value)
match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value)
if not len(match_multiple_dtstart):
raise serializers.ValidationError(_('Valid DTSTART required in rrule. Value should start with: DTSTART:YYYYMMDDTHHMMSSZ'))
if len(match_native_dtstart):
raise serializers.ValidationError(_('DTSTART cannot be a naive datetime. Specify ;TZINFO= or YYYYMMDDTHHMMSSZZ.'))
if len(match_multiple_dtstart) > 1:
raise serializers.ValidationError(_('Multiple DTSTART is not supported.'))
if not len(match_multiple_rrule):
raise serializers.ValidationError(_('RRULE required in rrule.'))
if len(match_multiple_rrule) > 1:
raise serializers.ValidationError(_('Multiple RRULE is not supported.'))
if 'interval' not in rrule_value.lower():
raise serializers.ValidationError(_('INTERVAL required in rrule.'))
if 'secondly' in rrule_value.lower():
raise serializers.ValidationError(_('SECONDLY is not supported.'))
if re.match(multi_by_month_day, rrule_value):
raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.'))
if re.match(multi_by_month, rrule_value):
raise serializers.ValidationError(_('Multiple BYMONTHs not supported.'))
if re.match(by_day_with_numeric_prefix, rrule_value):
raise serializers.ValidationError(_("BYDAY with numeric prefix not supported."))
if 'byyearday' in rrule_value.lower():
raise serializers.ValidationError(_("BYYEARDAY not supported."))
if 'byweekno' in rrule_value.lower():
raise serializers.ValidationError(_("BYWEEKNO not supported."))
if match_count:
count_val = match_count.groups()[0].strip().split("=")
if int(count_val[1]) > 999:
raise serializers.ValidationError(_("COUNT > 999 is unsupported."))
try:
Schedule.rrulestr(rrule_value)
except Exception:
raise serializers.ValidationError(_("rrule parsing failed validation."))
return value
class ScheduleSerializer(LaunchConfigurationBaseSerializer, SchedulePreviewSerializer):
show_capabilities = ['edit', 'delete']
class Meta:
@@ -3900,58 +3959,6 @@ class ScheduleSerializer(LaunchConfigurationBaseSerializer):
'Schedule its source project `{}` instead.').format(value.source_project.name)))
return value
# We reject rrules if:
# - DTSTART is not include
# - INTERVAL is not included
# - SECONDLY is used
# - TZID is used
# - BYDAY prefixed with a number (MO is good but not 20MO)
# - BYYEARDAY
# - BYWEEKNO
# - Multiple DTSTART or RRULE elements
# - COUNT > 999
def validate_rrule(self, value):
rrule_value = value
multi_by_month_day = ".*?BYMONTHDAY[\:\=][0-9]+,-*[0-9]+"
multi_by_month = ".*?BYMONTH[\:\=][0-9]+,[0-9]+"
by_day_with_numeric_prefix = ".*?BYDAY[\:\=][0-9]+[a-zA-Z]{2}"
match_count = re.match(".*?(COUNT\=[0-9]+)", rrule_value)
match_multiple_dtstart = re.findall(".*?(DTSTART\:[0-9]+T[0-9]+Z)", rrule_value)
match_multiple_rrule = re.findall(".*?(RRULE\:)", rrule_value)
if not len(match_multiple_dtstart):
raise serializers.ValidationError(_('DTSTART required in rrule. Value should match: DTSTART:YYYYMMDDTHHMMSSZ'))
if len(match_multiple_dtstart) > 1:
raise serializers.ValidationError(_('Multiple DTSTART is not supported.'))
if not len(match_multiple_rrule):
raise serializers.ValidationError(_('RRULE require in rrule.'))
if len(match_multiple_rrule) > 1:
raise serializers.ValidationError(_('Multiple RRULE is not supported.'))
if 'interval' not in rrule_value.lower():
raise serializers.ValidationError(_('INTERVAL required in rrule.'))
if 'tzid' in rrule_value.lower():
raise serializers.ValidationError(_('TZID is not supported.'))
if 'secondly' in rrule_value.lower():
raise serializers.ValidationError(_('SECONDLY is not supported.'))
if re.match(multi_by_month_day, rrule_value):
raise serializers.ValidationError(_('Multiple BYMONTHDAYs not supported.'))
if re.match(multi_by_month, rrule_value):
raise serializers.ValidationError(_('Multiple BYMONTHs not supported.'))
if re.match(by_day_with_numeric_prefix, rrule_value):
raise serializers.ValidationError(_("BYDAY with numeric prefix not supported."))
if 'byyearday' in rrule_value.lower():
raise serializers.ValidationError(_("BYYEARDAY not supported."))
if 'byweekno' in rrule_value.lower():
raise serializers.ValidationError(_("BYWEEKNO not supported."))
if match_count:
count_val = match_count.groups()[0].strip().split("=")
if int(count_val[1]) > 999:
raise serializers.ValidationError(_("COUNT > 999 is unsupported."))
try:
rrule.rrulestr(rrule_value)
except Exception:
raise serializers.ValidationError(_("rrule parsing failed validation."))
return value
class InstanceSerializer(BaseSerializer):

4
awx/api/urls/urls.py
View File

@@ -22,6 +22,8 @@ from awx.api.views import (
JobExtraCredentialsList,
JobTemplateCredentialsList,
JobTemplateExtraCredentialsList,
SchedulePreview,
ScheduleZoneInfo,
)
from .organization import urls as organization_urls
@@ -113,6 +115,8 @@ v2_urls = [
url(r'^jobs/(?P<pk>[0-9]+)/credentials/$', JobCredentialsList.as_view(), name='job_credentials_list'),
url(r'^job_templates/(?P<pk>[0-9]+)/extra_credentials/$', JobTemplateExtraCredentialsList.as_view(), name='job_template_extra_credentials_list'),
url(r'^job_templates/(?P<pk>[0-9]+)/credentials/$', JobTemplateCredentialsList.as_view(), name='job_template_credentials_list'),
url(r'^schedules/preview/$', SchedulePreview.as_view(), name='schedule_rrule'),
url(r'^schedules/zoneinfo/$', ScheduleZoneInfo.as_view(), name='schedule_zoneinfo'),
]
app_name = 'api'

38
awx/api/views.py
View File

@@ -53,6 +53,7 @@ import ansiconv
# Python Social Auth
from social_core.backends.utils import load_backends
import pytz
from wsgiref.util import FileWrapper
# AWX
@@ -608,6 +609,43 @@ class ScheduleDetail(RetrieveUpdateDestroyAPIView):
new_in_148 = True
class SchedulePreview(GenericAPIView):
model = Schedule
view_name = _('Schedule Recurrence Rule Preview')
serializer_class = SchedulePreviewSerializer
new_in_api_v2 = True
def post(self, request):
serializer = self.get_serializer(data=request.data)
if serializer.is_valid():
next_stamp = now()
schedule = []
gen = Schedule.rrulestr(serializer.validated_data['rrule']).xafter(next_stamp, count=20)
# loop across the entire generator and grab the first 10 events
for event in gen:
if len(schedule) >= 10:
break
if not dateutil.tz.datetime_exists(event):
# skip imaginary dates, like 2:30 on DST boundaries
continue
schedule.append(event)
return Response({
'local': schedule,
'utc': [s.astimezone(pytz.utc) for s in schedule]
})
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
class ScheduleZoneInfo(APIView):
def get(self, request):
from dateutil.zoneinfo import get_zonefile_instance
return Response(sorted(get_zonefile_instance().zones.keys()))
class LaunchConfigCredentialsBase(SubListAttachDetachAPIView):
model = Credential

99
awx/main/models/schedules.py
View File

@@ -5,11 +5,12 @@ import re
import logging
import datetime
import dateutil.rrule
from dateutil.tz import gettz, datetime_exists
# Django
from django.db import models
from django.db.models.query import QuerySet
from django.utils.timezone import now, make_aware, get_default_timezone
from django.utils.timezone import now, make_aware
from django.utils.translation import ugettext_lazy as _
# AWX
@@ -19,6 +20,9 @@ from awx.main.models.jobs import LaunchTimeConfig
from awx.main.utils import ignore_inventory_computed_fields
from awx.main.consumers import emit_channel_notification
import pytz
import six
logger = logging.getLogger('awx.main.models.schedule')
@@ -54,6 +58,10 @@ class ScheduleManager(ScheduleFilterMethods, models.Manager):
class Schedule(CommonModel, LaunchTimeConfig):
TZID_REGEX = re.compile(
"^(DTSTART;TZID=(?P<tzid>[^:]+)(?P<stamp>\:[0-9]+T[0-9]+))(?P<rrule> .*)$"
)
class Meta:
app_label = 'main'
ordering = ['-next_run']
@@ -92,6 +100,74 @@ class Schedule(CommonModel, LaunchTimeConfig):
help_text=_("The next time that the scheduled action will run.")
)
@classmethod
def rrulestr(cls, rrule, **kwargs):
"""
Apply our own custom rrule parsing logic. This applies some extensions
and limitations to parsing that are specific to our supported
implementation. Namely:
* python-dateutil doesn't _natively_ support `DTSTART;TZID=`; this
function parses out the TZID= component and uses it to produce the
`tzinfos` keyword argument to `dateutil.rrule.rrulestr()`. In this
way, we translate:
DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1
...into...
DTSTART:20180601T120000TZID RRULE:FREQ=DAILY;INTERVAL=1
...and we pass a hint about the local timezone to dateutil's parser:
`dateutil.rrule.rrulestr(rrule, {
'tzinfos': {
'TZID': dateutil.tz.gettz('America/New_York')
}
})`
it's possible that we can remove the custom code that performs this
parsing if TZID= gains support in upstream dateutil:
https://github.com/dateutil/dateutil/pull/615
* RFC5545 specifies that: if the "DTSTART" property is specified as
a date with local time (in our case, TZID=), then the UNTIL rule part
MUST also be treated as a date with local time. If the "DTSTART"
property is specified as a date with UTC time (with a Z suffix),
then the UNTIL rule part MUST be specified as a date with UTC time.
this function provides additional parsing to translate RRULES like this:
DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T170000
...into this (under the hood):
DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T210000Z
"""
source_rrule = rrule
kwargs['tzinfos'] = {}
match = cls.TZID_REGEX.match(rrule)
if match is not None:
rrule = cls.TZID_REGEX.sub("DTSTART\g<stamp>TZI\g<rrule>", rrule)
timezone = gettz(match.group('tzid'))
if 'until' in rrule.lower():
# if DTSTART;TZID= is used, coerce "naive" UNTIL values
# to the proper UTC date
match_until = re.match(".*?(?P<until>UNTIL\=[0-9]+T[0-9]+)(?P<utcflag>Z?)", rrule)
until_date = match_until.group('until').split("=")[1]
if len(match_until.group('utcflag')):
raise ValueError(six.text_type(
_('invalid rrule `{}` includes TZINFO= stanza and UTC-based UNTIL clause').format(source_rrule)
)) # noqa
localized = make_aware(
datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%S"),
timezone
)
utc = localized.astimezone(pytz.utc).strftime('%Y%m%dT%H%M%SZ')
rrule = rrule.replace(match_until.group('until'), 'UNTIL={}'.format(utc))
kwargs['tzinfos']['TZI'] = timezone
x = dateutil.rrule.rrulestr(rrule, **kwargs)
return x
def __unicode__(self):
return u'%s_t%s_%s_%s' % (self.name, self.unified_job_template.id, self.id, self.next_run)
@@ -107,21 +183,26 @@ class Schedule(CommonModel, LaunchTimeConfig):
return job_kwargs
def update_computed_fields(self):
future_rs = dateutil.rrule.rrulestr(self.rrule, forceset=True)
future_rs = Schedule.rrulestr(self.rrule, forceset=True)
next_run_actual = future_rs.after(now())
if next_run_actual is not None:
if not datetime_exists(next_run_actual):
# skip imaginary dates, like 2:30 on DST boundaries
next_run_actual = future_rs.after(next_run_actual)
next_run_actual = next_run_actual.astimezone(pytz.utc)
self.next_run = next_run_actual
try:
self.dtstart = future_rs[0]
self.dtstart = future_rs[0].astimezone(pytz.utc)
except IndexError:
self.dtstart = None
self.dtend = None
if 'until' in self.rrule.lower():
match_until = re.match(".*?(UNTIL\=[0-9]+T[0-9]+Z)", self.rrule)
until_date = match_until.groups()[0].split("=")[1]
self.dtend = make_aware(datetime.datetime.strptime(until_date, "%Y%m%dT%H%M%SZ"), get_default_timezone())
if 'count' in self.rrule.lower():
self.dtend = future_rs[-1]
if 'until' in self.rrule.lower() or 'count' in self.rrule.lower():
try:
self.dtend = future_rs[-1].astimezone(pytz.utc)
except IndexError:
self.dtend = None
emit_channel_notification('schedules-changed', dict(id=self.id, group_name='schedules'))
with ignore_inventory_computed_fields():
self.unified_job_template.update_computed_fields()

241
awx/main/tests/functional/api/test_schedules.py
View File

@@ -8,6 +8,17 @@ from awx.main.models import JobTemplate
RRULE_EXAMPLE = 'DTSTART:20151117T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1'
def get_rrule(tz=None):
parts = ['DTSTART']
if tz:
parts.append(';TZID={}'.format(tz))
parts.append(':20300308T050000')
if tz is None:
parts.append('Z')
parts.append(' RRULE:FREQ=DAILY;INTERVAL=1;COUNT=5')
return ''.join(parts)
@pytest.mark.django_db
def test_non_job_extra_vars_prohibited(post, project, admin_user):
url = reverse('api:project_schedules_list', kwargs={'pk': project.id})
@@ -32,3 +43,233 @@ def test_valid_survey_answer(post, admin_user, project, inventory, survey_spec_f
url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id})
post(url, {'name': 'test sch', 'rrule': RRULE_EXAMPLE, 'extra_data': '{"var1": 54}'},
admin_user, expect=201)
@pytest.mark.django_db
@pytest.mark.parametrize('rrule, error', [
("", "This field may not be blank"),
("DTSTART:NONSENSE", "Valid DTSTART required in rrule"),
("DTSTART:20300308T050000Z DTSTART:20310308T050000", "Multiple DTSTART is not supported"),
("DTSTART:20300308T050000Z", "RRULE required in rrule"),
("DTSTART:20300308T050000Z RRULE:NONSENSE", "INTERVAL required in rrule"),
("DTSTART:20300308T050000Z RRULE:FREQ=SECONDLY;INTERVAL=5;COUNT=6", "SECONDLY is not supported"),
("DTSTART:20300308T050000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=3,4", "Multiple BYMONTHDAYs not supported"), # noqa
("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=1,2", "Multiple BYMONTHs not supported"), # noqa
("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYDAY=5MO", "BYDAY with numeric prefix not supported"), # noqa
("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYYEARDAY=100", "BYYEARDAY not supported"), # noqa
("DTSTART:20300308T050000Z RRULE:FREQ=YEARLY;INTERVAL=1;BYWEEKNO=20", "BYWEEKNO not supported"),
("DTSTART:20300308T050000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=2000", "COUNT > 999 is unsupported"), # noqa
("DTSTART:20300308T050000Z RRULE:FREQ=REGULARLY;INTERVAL=1", "rrule parsing failed validation"), # noqa
("DTSTART;TZID=America/New_York:30200308T050000Z RRULE:FREQ=DAILY;INTERVAL=1", "rrule parsing failed validation"),
("DTSTART:20300308T050000 RRULE:FREQ=DAILY;INTERVAL=1", "DTSTART cannot be a naive datetime"),
])
def test_invalid_rrules(post, admin_user, project, inventory, rrule, error):
job_template = JobTemplate.objects.create(
name='test-jt',
project=project,
playbook='helloworld.yml',
inventory=inventory
)
url = reverse('api:job_template_schedules_list', kwargs={'pk': job_template.id})
resp = post(url, {
'name': 'Some Schedule',
'rrule': rrule,
}, admin_user, expect=400)
assert error in resp.content
@pytest.mark.django_db
def test_utc_preview(post, admin_user):
url = reverse('api:schedule_rrule')
r = post(url, {'rrule': get_rrule()}, admin_user, expect=200)
assert r.data['utc'] == r.data['local']
assert map(str, r.data['utc']) == [
'2030-03-08 05:00:00+00:00',
'2030-03-09 05:00:00+00:00',
'2030-03-10 05:00:00+00:00',
'2030-03-11 05:00:00+00:00',
'2030-03-12 05:00:00+00:00',
]
@pytest.mark.django_db
def test_nyc_with_dst(post, admin_user):
url = reverse('api:schedule_rrule')
r = post(url, {'rrule': get_rrule('America/New_York')}, admin_user, expect=200)
# March 10, 2030 is when DST takes effect in NYC
assert map(str, r.data['local']) == [
'2030-03-08 05:00:00-05:00',
'2030-03-09 05:00:00-05:00',
'2030-03-10 05:00:00-04:00',
'2030-03-11 05:00:00-04:00',
'2030-03-12 05:00:00-04:00',
]
assert map(str, r.data['utc']) == [
'2030-03-08 10:00:00+00:00',
'2030-03-09 10:00:00+00:00',
'2030-03-10 09:00:00+00:00',
'2030-03-11 09:00:00+00:00',
'2030-03-12 09:00:00+00:00',
]
@pytest.mark.django_db
def test_phoenix_without_dst(post, admin_user):
# The state of Arizona (aside from a few Native American territories) does
# not observe DST
url = reverse('api:schedule_rrule')
r = post(url, {'rrule': get_rrule('America/Phoenix')}, admin_user, expect=200)
# March 10, 2030 is when DST takes effect in NYC
assert map(str, r.data['local']) == [
'2030-03-08 05:00:00-07:00',
'2030-03-09 05:00:00-07:00',
'2030-03-10 05:00:00-07:00',
'2030-03-11 05:00:00-07:00',
'2030-03-12 05:00:00-07:00',
]
assert map(str, r.data['utc']) == [
'2030-03-08 12:00:00+00:00',
'2030-03-09 12:00:00+00:00',
'2030-03-10 12:00:00+00:00',
'2030-03-11 12:00:00+00:00',
'2030-03-12 12:00:00+00:00',
]
@pytest.mark.django_db
def test_interval_by_local_day(post, admin_user):
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
# March 10, 2030 is when DST takes effect in NYC
assert map(str, r.data['local']) == [
'2030-02-02 21:00:00-05:00',
'2030-03-02 21:00:00-05:00',
'2030-04-06 21:00:00-04:00',
'2030-05-04 21:00:00-04:00',
]
assert map(str, r.data['utc']) == [
'2030-02-03 02:00:00+00:00',
'2030-03-03 02:00:00+00:00',
'2030-04-07 01:00:00+00:00',
'2030-05-05 01:00:00+00:00',
]
@pytest.mark.django_db
def test_weekday_timezone_boundary(post, admin_user):
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20300101T210000 RRULE:FREQ=WEEKLY;BYDAY=TU;INTERVAL=1;COUNT=3'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
assert map(str, r.data['local']) == [
'2030-01-01 21:00:00-05:00',
'2030-01-08 21:00:00-05:00',
'2030-01-15 21:00:00-05:00',
]
assert map(str, r.data['utc']) == [
'2030-01-02 02:00:00+00:00',
'2030-01-09 02:00:00+00:00',
'2030-01-16 02:00:00+00:00',
]
@pytest.mark.django_db
def test_first_monthly_weekday_timezone_boundary(post, admin_user):
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20300101T210000 RRULE:FREQ=MONTHLY;BYDAY=SU;BYSETPOS=1;INTERVAL=1;COUNT=3'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
assert map(str, r.data['local']) == [
'2030-01-06 21:00:00-05:00',
'2030-02-03 21:00:00-05:00',
'2030-03-03 21:00:00-05:00',
]
assert map(str, r.data['utc']) == [
'2030-01-07 02:00:00+00:00',
'2030-02-04 02:00:00+00:00',
'2030-03-04 02:00:00+00:00',
]
@pytest.mark.django_db
def test_annual_timezone_boundary(post, admin_user):
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20301231T230000 RRULE:FREQ=YEARLY;INTERVAL=1;COUNT=3'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
assert map(str, r.data['local']) == [
'2030-12-31 23:00:00-05:00',
'2031-12-31 23:00:00-05:00',
'2032-12-31 23:00:00-05:00',
]
assert map(str, r.data['utc']) == [
'2031-01-01 04:00:00+00:00',
'2032-01-01 04:00:00+00:00',
'2033-01-01 04:00:00+00:00',
]
def test_dst_phantom_hour(post, admin_user):
# The DST period in the United States begins at 02:00 (2 am) local time, so
# the hour from 2:00:00 to 2:59:59 does not exist in the night of the
# switch.
# Three Sundays, starting 2:30AM America/New_York, starting Mar 3, 2030,
# should _not_ include Mar 10, 2030 @ 2:30AM (because it doesn't exist)
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20300303T023000 RRULE:FREQ=WEEKLY;BYDAY=SU;INTERVAL=1;COUNT=3'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
assert map(str, r.data['local']) == [
'2030-03-03 02:30:00-05:00',
'2030-03-17 02:30:00-04:00', # Skip 3/10 because 3/10 @ 2:30AM isn't a real date
]
assert map(str, r.data['utc']) == [
'2030-03-03 07:30:00+00:00',
'2030-03-17 06:30:00+00:00', # Skip 3/10 because 3/10 @ 2:30AM isn't a real date
]
@pytest.mark.django_db
def test_months_with_31_days(post, admin_user):
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20300101T000000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYMONTHDAY=31;COUNT=7'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
# 30 days have September, April, June, and November...
assert map(str, r.data['local']) == [
'2030-01-31 00:00:00-05:00',
'2030-03-31 00:00:00-04:00',
'2030-05-31 00:00:00-04:00',
'2030-07-31 00:00:00-04:00',
'2030-08-31 00:00:00-04:00',
'2030-10-31 00:00:00-04:00',
'2030-12-31 00:00:00-05:00',
]
def test_dst_rollback_duplicates(post, admin_user):
# From Nov 2 -> Nov 3, 2030, daylight savings ends and we "roll back" an hour.
# Make sure we don't "double count" duplicate times in the "rolled back"
# hour.
url = reverse('api:schedule_rrule')
rrule = 'DTSTART;TZID=America/New_York:20301102T233000 RRULE:FREQ=HOURLY;INTERVAL=1;COUNT=5'
r = post(url, {'rrule': rrule}, admin_user, expect=200)
assert map(str, r.data['local']) == [
'2030-11-02 23:30:00-04:00',
'2030-11-03 00:30:00-04:00',
'2030-11-03 01:30:00-04:00',
'2030-11-03 02:30:00-05:00',
'2030-11-03 03:30:00-05:00',
]

194
awx/main/tests/functional/models/test_schedule.py Normal file
View File

@@ -0,0 +1,194 @@
from datetime import datetime
import mock
import pytest
import pytz
from awx.main.models import JobTemplate, Schedule
@pytest.fixture
def job_template(inventory, project):
# need related resources set for these tests
return JobTemplate.objects.create(
name='test-job_template',
inventory=inventory,
project=project
)
@pytest.mark.django_db
def test_repeats_forever(job_template):
s = Schedule(
name='Some Schedule',
rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1',
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == str(s.dtstart) == '2030-01-12 21:00:00+00:00'
assert s.dtend is None
@pytest.mark.django_db
def test_no_recurrence_utc(job_template):
s = Schedule(
name='Some Schedule',
rrule='DTSTART:20300112T210000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1',
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-12 21:00:00+00:00'
@pytest.mark.django_db
def test_no_recurrence_est(job_template):
s = Schedule(
name='Some Schedule',
rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=1',
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == str(s.dtstart) == str(s.dtend) == '2030-01-13 02:00:00+00:00'
@pytest.mark.django_db
def test_next_run_utc(job_template):
s = Schedule(
name='Some Schedule',
rrule='DTSTART:20300112T210000Z RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4',
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2030-02-02 21:00:00+00:00'
assert str(s.next_run) == str(s.dtstart)
assert str(s.dtend) == '2030-05-04 21:00:00+00:00'
@pytest.mark.django_db
def test_next_run_est(job_template):
s = Schedule(
name='Some Schedule',
rrule='DTSTART;TZID=America/New_York:20300112T210000 RRULE:FREQ=MONTHLY;INTERVAL=1;BYDAY=SA;BYSETPOS=1;COUNT=4',
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2030-02-03 02:00:00+00:00'
assert str(s.next_run) == str(s.dtstart)
# March 10, 2030 is when DST takes effect in NYC
assert str(s.dtend) == '2030-05-05 01:00:00+00:00'
@pytest.mark.django_db
def test_year_boundary(job_template):
rrule = 'DTSTART;TZID=America/New_York:20301231T230000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=12;BYMONTHDAY=31;COUNT=4' # noqa
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2031-01-01 04:00:00+00:00' # UTC = +5 EST
assert str(s.next_run) == str(s.dtstart)
assert str(s.dtend) == '2034-01-01 04:00:00+00:00' # UTC = +5 EST
@pytest.mark.django_db
def test_leap_year_day(job_template):
rrule = 'DTSTART;TZID=America/New_York:20320229T050000 RRULE:FREQ=YEARLY;INTERVAL=1;BYMONTH=02;BYMONTHDAY=29;COUNT=2' # noqa
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2032-02-29 10:00:00+00:00' # UTC = +5 EST
assert str(s.next_run) == str(s.dtstart)
assert str(s.dtend) == '2036-02-29 10:00:00+00:00' # UTC = +5 EST
@pytest.mark.django_db
@pytest.mark.parametrize('until, dtend', [
['20180602T170000Z', '2018-06-02 12:00:00+00:00'],
['20180602T000000Z', '2018-06-01 12:00:00+00:00'],
])
def test_utc_until(job_template, until, dtend):
rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until)
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2018-06-01 12:00:00+00:00'
assert str(s.next_run) == str(s.dtstart)
assert str(s.dtend) == dtend
@pytest.mark.django_db
@pytest.mark.parametrize('until, dtend', [
['20180602T170000', '2018-06-02 16:00:00+00:00'],
['20180602T000000', '2018-06-01 16:00:00+00:00'],
])
def test_tzinfo_until(job_template, until, dtend):
rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL={}'.format(until) # noqa
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
assert str(s.next_run) == '2018-06-01 16:00:00+00:00' # UTC = +4 EST
assert str(s.next_run) == str(s.dtstart)
assert str(s.dtend) == dtend
@pytest.mark.django_db
def test_mismatched_until_timezone(job_template):
rrule = 'DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180602T000000' + 'Z' # noqa the Z isn't allowed, because we have a TZID=America/New_York
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
with pytest.raises(ValueError):
s.save()
@pytest.mark.django_db
def test_utc_until_in_the_past(job_template):
rrule = 'DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20150101T100000Z'
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
assert s.next_run is s.dtstart is s.dtend is None
@pytest.mark.django_db
@mock.patch('awx.main.models.schedules.now', lambda: datetime(2030, 03, 05, tzinfo=pytz.utc))
def test_dst_phantom_hour(job_template):
# The DST period in the United States begins at 02:00 (2 am) local time, so
# the hour from 2:00:00 to 2:59:59 does not exist in the night of the
# switch.
# Three Sundays, starting 2:30AM America/New_York, starting Mar 3, 2030,
# (which doesn't exist)
rrule = 'DTSTART;TZID=America/New_York:20300303T023000 RRULE:FREQ=WEEKLY;BYDAY=SU;INTERVAL=1;COUNT=3'
s = Schedule(
name='Some Schedule',
rrule=rrule,
unified_job_template=job_template
)
s.save()
# 3/10/30 @ 2:30AM is skipped because it _doesn't exist_ <cue twilight zone music>
assert str(s.next_run) == '2030-03-17 06:30:00+00:00'

145
docs/schedules.md Normal file
View File

@@ -0,0 +1,145 @@
Scheduled Jobs
==============
awx allows jobs to run on a schedule (with optional recurrence rules) via
an `HTTP POST` to a variety of API endpoints:
HTTP POST
https://tower-host.example.org/api/v2/job_templates/N/schedules/
https://tower-host.example.org/api/v2/projects/N/schedules/
https://tower-host.example.org/api/v2/inventory_sources/N/schedules/
https://tower-host.example.org/api/v2/system_jobs/N/schedules/
https://tower-host.example.org/api/v2/workflow_job_templates/N/schedules/
{
'name': 'My Schedule Name',
'rrule': 'DTSTART:20300115T120000Z RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7'
'extra_data': {}
}
...where `rrule` is a valid
[RFC5545](https://www.rfc-editor.org/rfc/rfc5545.txt) RRULE string. The
specific example above would run a job every day - for seven consecutive days - starting
on January 15th, 2030 at noon (UTC).
Specifying Timezones
====================
`DTSTART` values provided to awx _must_ provide timezone information (they may
not be naive dates).
For UTC dates, `DTSTART` values should be denoted with the `Z` suffix:
DTSTART:20300115T120000Z
Local timezones can be specified using the `TZID=` parameter:
DTSTART;TZID=America/New_York:20300115T120000
A list of _valid_ zone identifiers (which can vary by system) can be found at:
HTTP GET /api/v2/schedules/zoneinfo/
[
"Africa/Abidjan",
"Africa/Accra",
"Africa/Addis_Ababa",
...
]
UNTIL and Timezones
===================
RFC5545 specifies that:
> Furthermore, if the "DTSTART" property is specified as a date with local
> time, then the UNTIL rule part MUST also be specified as a date with local
> time. If the "DTSTART" property is specified as a date with UTC time or
> a date with local time and time zone reference, then the UNTIL rule part
> MUST be specified as a date with UTC time.
Given this, this RRULE:
`DTSTART:20180601T120000Z RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000Z`
...will be interpretted as "Starting on June 1st, 2018 at noon UTC, repeat
daily, ending on June 6th, 2018 at 5PM UTC".
This RRULE:
`DTSTART;TZID=America/New_York:20180601T120000 RRULE:FREQ=DAILY;INTERVAL=1;UNTIL=20180606T170000`
...will be interpretted as "Starting on June 1st, 2018 at noon EDT, repeat
daily, ending on June 6th, 2018 at 5PM EDT".
Previewing Schedules
====================
awx provides an endpoint for previewing the future dates and times for
a specified `RRULE`. A list of the next _ten_ occurrences will be returned in
local and UTC time:
POST https://tower-host.example.org/api/v2/schedules/preview/
{
'rrule': 'DTSTART;TZID=America/New_York:20300115T120000 RRULE:FREQ=DAILY;INTERVAL=1;COUNT=7'
}
Content-Type: application/json
{
"local": [
"2030-01-15T12:00:00-05:00",
"2030-01-16T12:00:00-05:00",
"2030-01-17T12:00:00-05:00",
"2030-01-18T12:00:00-05:00",
"2030-01-19T12:00:00-05:00",
"2030-01-20T12:00:00-05:00",
"2030-01-21T12:00:00-05:00"
],
"utc": [
"2030-01-15T17:00:00Z",
"2030-01-16T17:00:00Z",
"2030-01-17T17:00:00Z",
"2030-01-18T17:00:00Z",
"2030-01-19T17:00:00Z",
"2030-01-20T17:00:00Z",
"2030-01-21T17:00:00Z"
]
}
RRULE Limitations
=================
The following aspects of `RFC5545` are _not_ supported by awx schedules:
* Strings with more than a single `DTSTART:` component
* Strings with more than a single `RRULE` component
* The use of `FREQ=SECONDLY` in an `RRULE`
* The use of more than a single `FREQ=BYMONTHDAY` component in an `RRULE`
* The use of more than a single `FREQ=BYMONTHS` component in an `RRULE`
* The use of `FREQ=BYYEARDAY` in an `RRULE`
* The use of `FREQ=BYWEEKNO` in an `RRULE`
* The use of `FREQ=BYWEEKNO` in an `RRULE`
* The use of `COUNT=` in an `RRULE` with a value over 999
Implementation Details
======================
Any time an `awx.model.Schedule` is saved with a valid `rrule` value, the
`dateutil` library is used to burst out a list of all occurrences. From here,
the following dates are saved in the database:
* `main_schedule.rrule` - the original `RRULE` string provided by the user
* `main_schedule.dtstart` - the _first_ datetime in the list of all occurrences (coerced to UTC)
* `main_schedule.dtend` - the _last_ datetime in the list of all occurrences (coerced to UTC)
* `main_schedule.next_run` - the _next_ datetime in list after `utcnow()` (coerced to UTC)
awx makes use of [Celery Periodic Tasks
(celerybeat)](http://docs.celeryproject.org/en/latest/userguide/periodic-tasks.html)
to run a periodic task that discovers new jobs that need to run at a regular
interval (by default, every 30 seconds). When this task starts, it queries the
database for Schedules where `Schedule.next_run` is between
`scheduler_last_runtime()` and `utcnow()`. For each of these, a new job is
launched, and `Schedule.next_run` is changed to the next chronological datetime
in the list of all occurences.