awx/awx/main/models/jobs.py

# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

# Python
import codecs
import datetime
import logging
import os
import time
import json
from urllib.parse import urljoin


# Django
from django.conf import settings
from django.db import models
#from django.core.cache import cache
from django.utils.encoding import smart_str
from django.utils.timezone import now
from django.utils.translation import ugettext_lazy as _
from django.core.exceptions import FieldDoesNotExist

# REST Framework
from rest_framework.exceptions import ParseError

# AWX
from awx.api.versioning import reverse
from awx.main.models.base import (
    BaseModel, CreatedModifiedModel,
    prevent_search, accepts_json,
    JOB_TYPE_CHOICES, VERBOSITY_CHOICES,
    VarsDictProperty
)
from awx.main.models.events import JobEvent, SystemJobEvent
from awx.main.models.unified_jobs import (
    UnifiedJobTemplate, UnifiedJob
)
from awx.main.models.notifications import (
    NotificationTemplate,
    JobNotificationMixin,
)
from awx.main.utils import parse_yaml_or_json, getattr_dne, NullablePromptPseudoField
from awx.main.fields import ImplicitRoleField, JSONField, AskForField
from awx.main.models.mixins import (
    ResourceMixin,
    SurveyJobTemplateMixin,
    SurveyJobMixin,
    TaskManagerJobMixin,
    CustomVirtualEnvMixin,
    RelatedJobsMixin,
    WebhookMixin,
    WebhookTemplateMixin,
)


logger = logging.getLogger('awx.main.models.jobs')
analytics_logger = logging.getLogger('awx.analytics.job_events')
system_tracking_logger = logging.getLogger('awx.analytics.system_tracking')

__all__ = ['JobTemplate', 'JobLaunchConfig', 'Job', 'JobHostSummary', 'SystemJobTemplate', 'SystemJob']


class JobOptions(BaseModel):
    '''
    Common options for job templates and jobs.
    '''

    class Meta:
        abstract = True

    diff_mode = models.BooleanField(
        default=False,
        help_text=_("If enabled, textual changes made to any templated files on the host are shown in the standard output"),
    )
    job_type = models.CharField(
        max_length=64,
        choices=JOB_TYPE_CHOICES,
        default='run',
    )
    inventory = models.ForeignKey(
        'Inventory',
        related_name='%(class)ss',
        blank=True,
        null=True,
        default=None,
        on_delete=models.SET_NULL,
    )
    project = models.ForeignKey(
        'Project',
        related_name='%(class)ss',
        null=True,
        default=None,
        blank=True,
        on_delete=models.SET_NULL,
    )
    playbook = models.CharField(
        max_length=1024,
        default='',
        blank=True,
    )
    scm_branch = models.CharField(
        max_length=1024,
        default='',
        blank=True,
        help_text=_('Branch to use in job run. Project default used if blank. '
                    'Only allowed if project allow_override field is set to true.'),
    )
    forks = models.PositiveIntegerField(
        blank=True,
        default=0,
    )
    limit = models.TextField(
        blank=True,
        default='',
    )
    verbosity = models.PositiveIntegerField(
        choices=VERBOSITY_CHOICES,
        blank=True,
        default=0,
    )
    extra_vars = prevent_search(accepts_json(models.TextField(
        blank=True,
        default='',
    )))
    job_tags = models.CharField(
        max_length=1024,
        blank=True,
        default='',
    )
    force_handlers = models.BooleanField(
        blank=True,
        default=False,
    )
    skip_tags = models.CharField(
        max_length=1024,
        blank=True,
        default='',
    )
    start_at_task = models.CharField(
        max_length=1024,
        blank=True,
        default='',
    )
    become_enabled = models.BooleanField(
        default=False,
    )
    allow_simultaneous = models.BooleanField(
        default=False,
    )
    timeout = models.IntegerField(
        blank=True,
        default=0,
        help_text=_("The amount of time (in seconds) to run before the task is canceled."),
    )
    use_fact_cache = models.BooleanField(
        default=False,
        help_text=_(
            "If enabled, Tower will act as an Ansible Fact Cache Plugin; persisting "
            "facts at the end of a playbook run to the database and caching facts for use by Ansible."),
    )

    extra_vars_dict = VarsDictProperty('extra_vars', True)

    @property
    def machine_credential(self):
        return self.credentials.filter(credential_type__kind='ssh').first()

    @property
    def network_credentials(self):
        return list(self.credentials.filter(credential_type__kind='net'))

    @property
    def cloud_credentials(self):
        return list(self.credentials.filter(credential_type__kind='cloud'))

    @property
    def vault_credentials(self):
        return list(self.credentials.filter(credential_type__kind='vault'))

    @property
    def passwords_needed_to_start(self):
        '''Return list of password field names needed to start the job.'''
        needed = []
        # Unsaved credential objects can not require passwords
        if not self.pk:
            return needed
        for cred in self.credentials.all():
            needed.extend(cred.passwords_needed)
        return needed


class JobTemplate(UnifiedJobTemplate, JobOptions, SurveyJobTemplateMixin, ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin, WebhookTemplateMixin):
    '''
    A job template is a reusable job definition for applying a project (with
    playbook) to an inventory source with a given credential.
    '''
    FIELDS_TO_PRESERVE_AT_COPY = [
        'labels', 'instance_groups', 'credentials', 'survey_spec'
    ]
    FIELDS_TO_DISCARD_AT_COPY = ['vault_credential', 'credential']
    SOFT_UNIQUE_TOGETHER = [('polymorphic_ctype', 'name')]

    class Meta:
        app_label = 'main'
        ordering = ('name',)

    host_config_key = prevent_search(models.CharField(
        max_length=1024,
        blank=True,
        default='',
    ))
    ask_diff_mode_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_limit_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_tags_on_launch = AskForField(
        blank=True,
        default=False,
        allows_field='job_tags'
    )
    ask_skip_tags_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_job_type_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_verbosity_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_inventory_on_launch = AskForField(
        blank=True,
        default=False,
    )
    ask_credential_on_launch = AskForField(
        blank=True,
        default=False,
        allows_field='credentials'
    )
    ask_scm_branch_on_launch = AskForField(
        blank=True,
        default=False,
        allows_field='scm_branch'
    )
    job_slice_count = models.PositiveIntegerField(
        blank=True,
        default=1,
        help_text=_("The number of jobs to slice into at runtime. "
                    "Will cause the Job Template to launch a workflow if value is greater than 1."),
    )

    admin_role = ImplicitRoleField(
        parent_role=['project.organization.job_template_admin_role', 'inventory.organization.job_template_admin_role']
    )
    execute_role = ImplicitRoleField(
        parent_role=['admin_role', 'project.organization.execute_role', 'inventory.organization.execute_role'],
    )
    read_role = ImplicitRoleField(
        parent_role=['project.organization.auditor_role', 'inventory.organization.auditor_role', 'execute_role', 'admin_role'],
    )


    @classmethod
    def _get_unified_job_class(cls):
        return Job

    @classmethod
    def _get_unified_job_field_names(cls):
        return set(f.name for f in JobOptions._meta.fields) | set(
            ['name', 'description', 'survey_passwords', 'labels', 'credentials',
             'job_slice_number', 'job_slice_count']
        )

    @property
    def validation_errors(self):
        '''
        Fields needed to start, which cannot be given on launch, invalid state.
        '''
        validation_errors = {}
        if self.inventory is None and not self.ask_inventory_on_launch:
            validation_errors['inventory'] = [_("Job Template must provide 'inventory' or allow prompting for it."),]
        if self.project is None:
            validation_errors['project'] = [_("Job Templates must have a project assigned."),]
        return validation_errors

    @property
    def resources_needed_to_start(self):
        return [fd for fd in ['project', 'inventory'] if not getattr(self, '{}_id'.format(fd))]

    def create_job(self, **kwargs):
        '''
        Create a new job based on this template.
        '''
        return self.create_unified_job(**kwargs)

    def get_effective_slice_ct(self, kwargs):
        actual_inventory = self.inventory
        if self.ask_inventory_on_launch and 'inventory' in kwargs:
            actual_inventory = kwargs['inventory']
        if actual_inventory:
            return min(self.job_slice_count, actual_inventory.hosts.count())
        else:
            return self.job_slice_count

    def create_unified_job(self, **kwargs):
        prevent_slicing = kwargs.pop('_prevent_slicing', False)
        slice_ct = self.get_effective_slice_ct(kwargs)
        slice_event = bool(slice_ct > 1 and (not prevent_slicing))
        if slice_event:
            # A Slice Job Template will generate a WorkflowJob rather than a Job
            from awx.main.models.workflow import WorkflowJobTemplate, WorkflowJobNode
            kwargs['_unified_job_class'] = WorkflowJobTemplate._get_unified_job_class()
            kwargs['_parent_field_name'] = "job_template"
            kwargs.setdefault('_eager_fields', {})
            kwargs['_eager_fields']['is_sliced_job'] = True
        elif self.job_slice_count > 1 and (not prevent_slicing):
            # Unique case where JT was set to slice but hosts not available
            kwargs.setdefault('_eager_fields', {})
            kwargs['_eager_fields']['job_slice_count'] = 1
        elif prevent_slicing:
            kwargs.setdefault('_eager_fields', {})
            kwargs['_eager_fields'].setdefault('job_slice_count', 1)
        job = super(JobTemplate, self).create_unified_job(**kwargs)
        if slice_event:
            for idx in range(slice_ct):
                create_kwargs = dict(workflow_job=job,
                                     unified_job_template=self,
                                     ancestor_artifacts=dict(job_slice=idx + 1))
                WorkflowJobNode.objects.create(**create_kwargs)
        return job

    def get_absolute_url(self, request=None):
        return reverse('api:job_template_detail', kwargs={'pk': self.pk}, request=request)

    def can_start_without_user_input(self, callback_extra_vars=None):
        '''
        Return whether job template can be used to start a new job without
        requiring any user input.
        '''
        variables_needed = False
        if callback_extra_vars:
            extra_vars_dict = parse_yaml_or_json(callback_extra_vars)
            for var in self.variables_needed_to_start:
                if var not in extra_vars_dict:
                    variables_needed = True
                    break
        elif self.variables_needed_to_start:
            variables_needed = True
        prompting_needed = False
        # The behavior of provisioning callback should mimic
        # that of job template launch, so prompting_needed should
        # not block a provisioning callback from creating/launching jobs.
        if callback_extra_vars is None:
            for ask_field_name in set(self.get_ask_mapping().values()):
                if getattr(self, ask_field_name):
                    prompting_needed = True
                    break
        return (not prompting_needed and
                not self.passwords_needed_to_start and
                not variables_needed)

    def _accept_or_ignore_job_kwargs(self, **kwargs):
        exclude_errors = kwargs.pop('_exclude_errors', [])
        prompted_data = {}
        rejected_data = {}
        accepted_vars, rejected_vars, errors_dict = self.accept_or_ignore_variables(
            kwargs.get('extra_vars', {}),
            _exclude_errors=exclude_errors,
            extra_passwords=kwargs.get('survey_passwords', {}))
        if accepted_vars:
            prompted_data['extra_vars'] = accepted_vars
        if rejected_vars:
            rejected_data['extra_vars'] = rejected_vars

        # Handle all the other fields that follow the simple prompting rule
        for field_name, ask_field_name in self.get_ask_mapping().items():
            if field_name not in kwargs or field_name == 'extra_vars' or kwargs[field_name] is None:
                continue

            new_value = kwargs[field_name]
            old_value = getattr(self, field_name)

            field = self._meta.get_field(field_name)
            if isinstance(field, models.ManyToManyField):
                old_value = set(old_value.all())
                if getattr(self, '_deprecated_credential_launch', False):
                    # TODO: remove this code branch when support for `extra_credentials` goes away
                    new_value = set(kwargs[field_name])
                else:
                    new_value = set(kwargs[field_name]) - old_value
                    if not new_value:
                        continue

            if new_value == old_value:
                # no-op case: Fields the same as template's value
                # counted as neither accepted or ignored
                continue
            elif field_name == 'scm_branch' and old_value == '' and self.project and new_value == self.project.scm_branch:
                # special case of "not provided" for branches
                # job template does not provide branch, runs with default branch
                continue
            elif getattr(self, ask_field_name):
                # Special case where prompts can be rejected based on project setting
                if field_name == 'scm_branch':
                    if not self.project:
                        rejected_data[field_name] = new_value
                        errors_dict[field_name] = _('Project is missing.')
                        continue
                    if kwargs['scm_branch'] != self.project.scm_branch and not self.project.allow_override:
                        rejected_data[field_name] = new_value
                        errors_dict[field_name] = _('Project does not allow override of branch.')
                        continue
                # accepted prompt
                prompted_data[field_name] = new_value
            else:
                # unprompted - template is not configured to accept field on launch
                rejected_data[field_name] = new_value
                # Not considered an error for manual launch, to support old
                # behavior of putting them in ignored_fields and launching anyway
                if 'prompts' not in exclude_errors:
                    errors_dict[field_name] = _('Field is not configured to prompt on launch.')

        if ('prompts' not in exclude_errors and
                (not getattr(self, 'ask_credential_on_launch', False)) and
                self.passwords_needed_to_start):
            errors_dict['passwords_needed_to_start'] = _(
                'Saved launch configurations cannot provide passwords needed to start.')

        needed = self.resources_needed_to_start
        if needed:
            needed_errors = []
            for resource in needed:
                if resource in prompted_data:
                    continue
                needed_errors.append(_("Job Template {} is missing or undefined.").format(resource))
            if needed_errors:
                errors_dict['resources_needed_to_start'] = needed_errors

        return prompted_data, rejected_data, errors_dict

    @property
    def cache_timeout_blocked(self):
        if Job.objects.filter(job_template=self, status__in=['pending', 'waiting', 'running']).count() >= getattr(settings, 'SCHEDULE_MAX_JOBS', 10):
            logger.error("Job template %s could not be started because there are more than %s other jobs from that template waiting to run" %
                         (self.name, getattr(settings, 'SCHEDULE_MAX_JOBS', 10)))
            return True
        return False

    def _can_update(self):
        return self.can_start_without_user_input()

    @property
    def notification_templates(self):
        # Return all notification_templates defined on the Job Template, on the Project, and on the Organization for each trigger type
        # TODO: Currently there is no org fk on project so this will need to be added once that is
        #       available after the rbac pr
        base_notification_templates = NotificationTemplate.objects
        error_notification_templates = list(base_notification_templates.filter(
            unifiedjobtemplate_notification_templates_for_errors__in=[self, self.project]))
        started_notification_templates = list(base_notification_templates.filter(
            unifiedjobtemplate_notification_templates_for_started__in=[self, self.project]))
        success_notification_templates = list(base_notification_templates.filter(
            unifiedjobtemplate_notification_templates_for_success__in=[self, self.project]))
        # Get Organization NotificationTemplates
        if self.project is not None and self.project.organization is not None:
            error_notification_templates = set(error_notification_templates + list(base_notification_templates.filter(
                organization_notification_templates_for_errors=self.project.organization)))
            started_notification_templates = set(started_notification_templates + list(base_notification_templates.filter(
                organization_notification_templates_for_started=self.project.organization)))
            success_notification_templates = set(success_notification_templates + list(base_notification_templates.filter(
                organization_notification_templates_for_success=self.project.organization)))
        return dict(error=list(error_notification_templates),
                    started=list(started_notification_templates),
                    success=list(success_notification_templates))

    '''
    RelatedJobsMixin
    '''
    def _get_related_jobs(self):
        return UnifiedJob.objects.filter(unified_job_template=self)


class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskManagerJobMixin, CustomVirtualEnvMixin, WebhookMixin):
    '''
    A job applies a project (with playbook) to an inventory source with a given
    credential.  It represents a single invocation of ansible-playbook with the
    given parameters.
    '''

    class Meta:
        app_label = 'main'
        ordering = ('id',)

    job_template = models.ForeignKey(
        'JobTemplate',
        related_name='jobs',
        blank=True,
        null=True,
        default=None,
        on_delete=models.SET_NULL,
    )
    hosts = models.ManyToManyField(
        'Host',
        related_name='jobs',
        editable=False,
        through='JobHostSummary',
    )
    artifacts = JSONField(
        blank=True,
        default=dict,
        editable=False,
    )
    scm_revision = models.CharField(
        max_length=1024,
        blank=True,
        default='',
        editable=False,
        verbose_name=_('SCM Revision'),
        help_text=_('The SCM Revision from the Project used for this job, if available'),
    )
    project_update = models.ForeignKey(
        'ProjectUpdate',
        blank=True,
        null=True,
        default=None,
        on_delete=models.SET_NULL,
        help_text=_('The SCM Refresh task used to make sure the playbooks were available for the job run'),
    )
    job_slice_number = models.PositiveIntegerField(
        blank=True,
        default=0,
        help_text=_("If part of a sliced job, the ID of the inventory slice operated on. "
                    "If not part of sliced job, parameter is not used."),
    )
    job_slice_count = models.PositiveIntegerField(
        blank=True,
        default=1,
        help_text=_("If ran as part of sliced jobs, the total number of slices. "
                    "If 1, job is not part of a sliced job."),
    )


    def _get_parent_field_name(self):
        return 'job_template'

    @classmethod
    def _get_task_class(cls):
        from awx.main.tasks import RunJob
        return RunJob

    @classmethod
    def supports_isolation(cls):
        return True

    def _global_timeout_setting(self):
        return 'DEFAULT_JOB_TIMEOUT'

    @classmethod
    def _get_unified_job_template_class(cls):
        return JobTemplate

    def get_absolute_url(self, request=None):
        return reverse('api:job_detail', kwargs={'pk': self.pk}, request=request)

    def get_ui_url(self):
        return urljoin(settings.TOWER_URL_BASE, "/#/jobs/playbook/{}".format(self.pk))

    @property
    def ansible_virtualenv_path(self):
        # the order here enforces precedence (it matters)
        for virtualenv in (
            self.job_template.custom_virtualenv if self.job_template else None,
            self.project.custom_virtualenv,
            self.project.organization.custom_virtualenv if self.project.organization else None
        ):
            if virtualenv:
                return virtualenv
        return settings.ANSIBLE_VENV_PATH

    @property
    def event_class(self):
        return JobEvent

    def copy_unified_job(self, **new_prompts):
        # Needed for job slice relaunch consistency, do no re-spawn workflow job
        # target same slice as original job
        new_prompts['_prevent_slicing'] = True
        new_prompts.setdefault('_eager_fields', {})
        new_prompts['_eager_fields']['job_slice_number'] = self.job_slice_number
        new_prompts['_eager_fields']['job_slice_count'] = self.job_slice_count
        return super(Job, self).copy_unified_job(**new_prompts)

    def get_passwords_needed_to_start(self):
        return self.passwords_needed_to_start

    def _get_hosts(self, **kwargs):
        Host = JobHostSummary._meta.get_field('host').related_model
        kwargs['job_host_summaries__job__pk'] = self.pk
        return Host.objects.filter(**kwargs)

    def retry_qs(self, status):
        '''
        Returns Host queryset that will be used to produce the `limit`
        field in a retry on a subset of hosts
        '''
        kwargs = {}
        if status == 'all':
            pass
        elif status == 'failed':
            # Special case for parity with Ansible .retry files
            kwargs['job_host_summaries__failed'] = True
        elif status in ['ok', 'changed', 'unreachable']:
            if status == 'unreachable':
                status_field = 'dark'
            else:
                status_field = status
            kwargs['job_host_summaries__{}__gt'.format(status_field)] = 0
        else:
            raise ParseError(_(
                '{status_value} is not a valid status option.'
            ).format(status_value=status))
        return self._get_hosts(**kwargs)

    @property
    def task_impact(self):
        if self.launch_type == 'callback':
            count_hosts = 2
        else:
            # If for some reason we can't count the hosts then lets assume the impact as forks
            if self.inventory is not None:
                count_hosts = self.inventory.hosts.count()
                if self.job_slice_count > 1:
                    # Integer division intentional
                    count_hosts = (count_hosts + self.job_slice_count - self.job_slice_number) // self.job_slice_count
            else:
                count_hosts = 5 if self.forks == 0 else self.forks
        return min(count_hosts, 5 if self.forks == 0 else self.forks) + 1

    @property
    def successful_hosts(self):
        return self._get_hosts(job_host_summaries__ok__gt=0)

    @property
    def failed_hosts(self):
        return self._get_hosts(job_host_summaries__failures__gt=0)

    @property
    def changed_hosts(self):
        return self._get_hosts(job_host_summaries__changed__gt=0)

    @property
    def dark_hosts(self):
        return self._get_hosts(job_host_summaries__dark__gt=0)

    @property
    def unreachable_hosts(self):
        return self.dark_hosts

    @property
    def skipped_hosts(self):
        return self._get_hosts(job_host_summaries__skipped__gt=0)

    @property
    def processed_hosts(self):
        return self._get_hosts(job_host_summaries__processed__gt=0)

    @property
    def ignored_hosts(self):
        return self._get_hosts(job_host_summaries__ignored__gt=0)

    @property
    def rescued_hosts(self):
        return self._get_hosts(job_host_summaries__rescued__gt=0)

    def notification_data(self, block=5):
        data = super(Job, self).notification_data()
        all_hosts = {}
        # NOTE: Probably related to job event slowness, remove at some point -matburt
        if block and self.status != 'running':
            summaries = self.job_host_summaries.all()
            while block > 0 and not len(summaries):
                time.sleep(1)
                block -= 1
        else:
            summaries = self.job_host_summaries.all()
        for h in self.job_host_summaries.all():
            all_hosts[h.host_name] = dict(failed=h.failed,
                                          changed=h.changed,
                                          dark=h.dark,
                                          failures=h.failures,
                                          ok=h.ok,
                                          processed=h.processed,
                                          skipped=h.skipped,
                                          rescued=h.rescued,
                                          ignored=h.ignored)
        data.update(dict(inventory=self.inventory.name if self.inventory else None,
                         project=self.project.name if self.project else None,
                         playbook=self.playbook,
                         credential=getattr(self.machine_credential, 'name', None),
                         limit=self.limit,
                         extra_vars=self.display_extra_vars(),
                         hosts=all_hosts))
        return data

    def _resources_sufficient_for_launch(self):
        return not (self.inventory_id is None or self.project_id is None)

    def display_artifacts(self):
        '''
        Hides artifacts if they are marked as no_log type artifacts.
        '''
        artifacts = self.artifacts
        if artifacts.get('_ansible_no_log', False):
            return "$hidden due to Ansible no_log flag$"
        return artifacts

    @property
    def can_run_containerized(self):
        return any([ig for ig in self.preferred_instance_groups if ig.is_containerized])

    @property
    def is_containerized(self):
        return bool(self.instance_group and self.instance_group.is_containerized)

    @property
    def preferred_instance_groups(self):
        if self.project is not None and self.project.organization is not None:
            organization_groups = [x for x in self.project.organization.instance_groups.all()]
        else:
            organization_groups = []
        if self.inventory is not None:
            inventory_groups = [x for x in self.inventory.instance_groups.all()]
        else:
            inventory_groups = []
        if self.job_template is not None:
            template_groups = [x for x in self.job_template.instance_groups.all()]
        else:
            template_groups = []
        selected_groups = template_groups + inventory_groups + organization_groups
        if not selected_groups:
            return self.global_instance_groups
        return selected_groups

    def awx_meta_vars(self):
        r = super(Job, self).awx_meta_vars()
        if self.project:
            for name in ('awx', 'tower'):
                r['{}_project_revision'.format(name)] = self.project.scm_revision
        if self.job_template:
            for name in ('awx', 'tower'):
                r['{}_job_template_id'.format(name)] = self.job_template.pk
                r['{}_job_template_name'.format(name)] = self.job_template.name
        return r

    '''
    JobNotificationMixin
    '''
    def get_notification_templates(self):
        if not self.job_template:
            return NotificationTemplate.objects.none()
        return self.job_template.notification_templates

    def get_notification_friendly_name(self):
        return "Job"

    def _get_inventory_hosts(
        self,
        only=['name', 'ansible_facts', 'ansible_facts_modified', 'modified', 'inventory_id']
    ):
        if not self.inventory:
            return []
        return self.inventory.hosts.only(*only)

    def start_job_fact_cache(self, destination, modification_times, timeout=None):
        os.makedirs(destination, mode=0o700)
        hosts = self._get_inventory_hosts()
        if timeout is None:
            timeout = settings.ANSIBLE_FACT_CACHE_TIMEOUT
        if timeout > 0:
            # exclude hosts with fact data older than `settings.ANSIBLE_FACT_CACHE_TIMEOUT seconds`
            timeout = now() - datetime.timedelta(seconds=timeout)
            hosts = hosts.filter(ansible_facts_modified__gte=timeout)
        for host in hosts:
            filepath = os.sep.join(map(str, [destination, host.name]))
            if not os.path.realpath(filepath).startswith(destination):
                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
                continue
            try:
                with codecs.open(filepath, 'w', encoding='utf-8') as f:
                    os.chmod(f.name, 0o600)
                    json.dump(host.ansible_facts, f)
            except IOError:
                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
                continue
            # make note of the time we wrote the file so we can check if it changed later
            modification_times[filepath] = os.path.getmtime(filepath)

    def finish_job_fact_cache(self, destination, modification_times):
        for host in self._get_inventory_hosts():
            filepath = os.sep.join(map(str, [destination, host.name]))
            if not os.path.realpath(filepath).startswith(destination):
                system_tracking_logger.error('facts for host {} could not be cached'.format(smart_str(host.name)))
                continue
            if os.path.exists(filepath):
                # If the file changed since we wrote it pre-playbook run...
                modified = os.path.getmtime(filepath)
                if modified > modification_times.get(filepath, 0):
                    with codecs.open(filepath, 'r', encoding='utf-8') as f:
                        try:
                            ansible_facts = json.load(f)
                        except ValueError:
                            continue
                        host.ansible_facts = ansible_facts
                        host.ansible_facts_modified = now()
                        ansible_local_system_id = ansible_facts.get('ansible_local', {}).get('insights', {}).get('system_id', None)
                        ansible_facts_system_id = ansible_facts.get('insights', {}).get('system_id', None)
                        if ansible_local_system_id:
                            print("Setting local {}".format(ansible_local_system_id))
                            logger.debug("Insights system_id {} found for host <{}, {}> in"
                                         " ansible local facts".format(ansible_local_system_id,
                                                                       host.inventory.id,
                                                                       host.name))
                            host.insights_system_id = ansible_local_system_id
                        elif ansible_facts_system_id:
                            logger.debug("Insights system_id {} found for host <{}, {}> in"
                                         " insights facts".format(ansible_local_system_id,
                                                                  host.inventory.id,
                                                                  host.name))
                            host.insights_system_id = ansible_facts_system_id
                        host.save()
                        system_tracking_logger.info(
                            'New fact for inventory {} host {}'.format(
                                smart_str(host.inventory.name), smart_str(host.name)),
                            extra=dict(inventory_id=host.inventory.id, host_name=host.name,
                                       ansible_facts=host.ansible_facts,
                                       ansible_facts_modified=host.ansible_facts_modified.isoformat(),
                                       job_id=self.id))
            else:
                # if the file goes missing, ansible removed it (likely via clear_facts)
                host.ansible_facts = {}
                host.ansible_facts_modified = now()
                system_tracking_logger.info(
                    'Facts cleared for inventory {} host {}'.format(
                        smart_str(host.inventory.name), smart_str(host.name)))
                host.save()


class LaunchTimeConfigBase(BaseModel):
    '''
    Needed as separate class from LaunchTimeConfig because some models
    use `extra_data` and some use `extra_vars`. We cannot change the API,
    so we force fake it in the model definitions
     - model defines extra_vars - use this class
     - model needs to use extra data - use LaunchTimeConfig
    Use this for models which are SurveyMixins and UnifiedJobs or Templates
    '''
    class Meta:
        abstract = True

    # Prompting-related fields that have to be handled as special cases
    inventory = models.ForeignKey(
        'Inventory',
        related_name='%(class)ss',
        blank=True,
        null=True,
        default=None,
        on_delete=models.SET_NULL,
        help_text=_('Inventory applied as a prompt, assuming job template prompts for inventory')
    )
    # All standard fields are stored in this dictionary field
    # This is a solution to the nullable CharField problem, specific to prompting
    char_prompts = JSONField(
        blank=True,
        default=dict
    )

    def prompts_dict(self, display=False):
        data = {}
        # Some types may have different prompts, but always subset of JT prompts
        for prompt_name in JobTemplate.get_ask_mapping().keys():
            try:
                field = self._meta.get_field(prompt_name)
            except FieldDoesNotExist:
                field = None
            if isinstance(field, models.ManyToManyField):
                if not self.pk:
                    continue  # unsaved object can't have related many-to-many
                prompt_val = set(getattr(self, prompt_name).all())
                if len(prompt_val) > 0:
                    data[prompt_name] = prompt_val
            elif prompt_name == 'extra_vars':
                if self.extra_vars:
                    if display:
                        data[prompt_name] = self.display_extra_vars()
                    else:
                        data[prompt_name] = self.extra_vars
                if self.survey_passwords and not display:
                    data['survey_passwords'] = self.survey_passwords
            else:
                prompt_val = getattr(self, prompt_name)
                if prompt_val is not None:
                    data[prompt_name] = prompt_val
        return data


for field_name in JobTemplate.get_ask_mapping().keys():
    if field_name == 'extra_vars':
        continue
    try:
        LaunchTimeConfigBase._meta.get_field(field_name)
    except FieldDoesNotExist:
        setattr(LaunchTimeConfigBase, field_name, NullablePromptPseudoField(field_name))


class LaunchTimeConfig(LaunchTimeConfigBase):
    '''
    Common model for all objects that save details of a saved launch config
    WFJT / WJ nodes, schedules, and job launch configs (not all implemented yet)
    '''
    class Meta:
        abstract = True

    # Special case prompting fields, even more special than the other ones
    extra_data = JSONField(
        blank=True,
        default=dict
    )
    survey_passwords = prevent_search(JSONField(
        blank=True,
        default=dict,
        editable=False,
    ))
    # Credentials needed for non-unified job / unified JT models
    credentials = models.ManyToManyField(
        'Credential',
        related_name='%(class)ss'
    )

    @property
    def extra_vars(self):
        return self.extra_data

    @extra_vars.setter
    def extra_vars(self, extra_vars):
        self.extra_data = extra_vars

    def display_extra_vars(self):
        '''
        Hides fields marked as passwords in survey.
        '''
        if hasattr(self, 'survey_passwords') and self.survey_passwords:
            extra_vars = parse_yaml_or_json(self.extra_vars).copy()
            for key, value in self.survey_passwords.items():
                if key in extra_vars:
                    extra_vars[key] = value
            return extra_vars
        else:
            return self.extra_vars

    def display_extra_data(self):
        return self.display_extra_vars()


class JobLaunchConfig(LaunchTimeConfig):
    '''
    Historical record of user launch-time overrides for a job
    Not exposed in the API
    Used for relaunch, scheduling, etc.
    '''
    class Meta:
        app_label = 'main'

    job = models.OneToOneField(
        'UnifiedJob',
        related_name='launch_config',
        on_delete=models.CASCADE,
        editable=False,
    )

    def has_user_prompts(self, template):
        '''
        Returns True if any fields exist in the launch config that are
        not permissions exclusions
        (has to exist because of callback relaunch exception)
        '''
        return self._has_user_prompts(template, only_unprompted=False)

    def has_unprompted(self, template):
        '''
        returns True if the template has set ask_ fields to False after
        launching with those prompts
        '''
        return self._has_user_prompts(template, only_unprompted=True)

    def _has_user_prompts(self, template, only_unprompted=True):
        prompts = self.prompts_dict()
        ask_mapping = template.get_ask_mapping()
        if template.survey_enabled and (not template.ask_variables_on_launch):
            ask_mapping.pop('extra_vars')
            provided_vars = set(prompts.get('extra_vars', {}).keys())
            survey_vars = set(
                element.get('variable') for element in
                template.survey_spec.get('spec', {}) if 'variable' in element
            )
            if (provided_vars and not only_unprompted) or (provided_vars - survey_vars):
                return True
        for field_name, ask_field_name in ask_mapping.items():
            if field_name in prompts and not (getattr(template, ask_field_name) and only_unprompted):
                if field_name == 'limit' and self.job and self.job.launch_type == 'callback':
                    continue  # exception for relaunching callbacks
                return True
        else:
            return False


class JobHostSummary(CreatedModifiedModel):
    '''
    Per-host statistics for each job.
    '''

    class Meta:
        app_label = 'main'
        unique_together = [('job', 'host_name')]
        verbose_name_plural = _('job host summaries')
        ordering = ('-pk',)

    job = models.ForeignKey(
        'Job',
        related_name='job_host_summaries',
        on_delete=models.CASCADE,
        editable=False,
    )
    host = models.ForeignKey('Host',
                             related_name='job_host_summaries',
                             null=True,
                             default=None,
                             on_delete=models.SET_NULL,
                             editable=False)

    host_name = models.CharField(
        max_length=1024,
        default='',
        editable=False,
    )

    changed = models.PositiveIntegerField(default=0, editable=False)
    dark = models.PositiveIntegerField(default=0, editable=False)
    failures = models.PositiveIntegerField(default=0, editable=False)
    ignored = models.PositiveIntegerField(default=0, editable=False)
    ok = models.PositiveIntegerField(default=0, editable=False)
    processed = models.PositiveIntegerField(default=0, editable=False)
    rescued = models.PositiveIntegerField(default=0, editable=False)
    skipped = models.PositiveIntegerField(default=0, editable=False)
    failed = models.BooleanField(default=False, editable=False)

    def __str__(self):
        host = getattr_dne(self, 'host')
        hostname = host.name if host else 'N/A'
        return '%s changed=%d dark=%d failures=%d ignored=%d ok=%d processed=%d rescued=%d skipped=%s' % \
            (hostname, self.changed, self.dark, self.failures, self.ignored, self.ok,
             self.processed, self.rescued, self.skipped)

    def get_absolute_url(self, request=None):
        return reverse('api:job_host_summary_detail', kwargs={'pk': self.pk}, request=request)

    def save(self, *args, **kwargs):
        # If update_fields has been specified, add our field names to it,
        # if it hasn't been specified, then we're just doing a normal save.
        if self.host is not None:
            self.host_name = self.host.name
        update_fields = kwargs.get('update_fields', [])
        self.failed = bool(self.dark or self.failures)
        update_fields.append('failed')
        super(JobHostSummary, self).save(*args, **kwargs)
        self.update_host_last_job_summary()

    def update_host_last_job_summary(self):
        update_fields = []
        if self.host is None:
            return
        if self.host.last_job_id != self.job_id:
            self.host.last_job_id = self.job_id
            update_fields.append('last_job_id')
        if self.host.last_job_host_summary_id != self.id:
            self.host.last_job_host_summary_id = self.id
            update_fields.append('last_job_host_summary_id')
        if update_fields:
            self.host.save(update_fields=update_fields)
        #self.host.update_computed_fields()


class SystemJobOptions(BaseModel):
    '''
    Common fields for SystemJobTemplate and SystemJob.
    '''

    SYSTEM_JOB_TYPE = [
        ('cleanup_jobs', _('Remove jobs older than a certain number of days')),
        ('cleanup_activitystream', _('Remove activity stream entries older than a certain number of days')),
        ('clearsessions', _('Removes expired browser sessions from the database')),
        ('cleartokens', _('Removes expired OAuth 2 access tokens and refresh tokens'))
    ]

    class Meta:
        abstract = True

    job_type = models.CharField(
        max_length=32,
        choices=SYSTEM_JOB_TYPE,
        blank=True,
        default='',
    )


class SystemJobTemplate(UnifiedJobTemplate, SystemJobOptions):

    class Meta:
        app_label = 'main'

    @classmethod
    def _get_unified_job_class(cls):
        return SystemJob

    @classmethod
    def _get_unified_job_field_names(cls):
        return ['name', 'description', 'job_type', 'extra_vars']

    def get_absolute_url(self, request=None):
        return reverse('api:system_job_template_detail', kwargs={'pk': self.pk}, request=request)

    @property
    def cache_timeout_blocked(self):
        return False

    @property
    def notification_templates(self):
        # TODO: Go through RBAC instead of calling all(). Need to account for orphaned NotificationTemplates
        base_notification_templates = NotificationTemplate.objects.all()
        error_notification_templates = list(base_notification_templates
                                            .filter(unifiedjobtemplate_notification_templates_for_errors__in=[self]))
        started_notification_templates = list(base_notification_templates
                                              .filter(unifiedjobtemplate_notification_templates_for_started__in=[self]))
        success_notification_templates = list(base_notification_templates
                                              .filter(unifiedjobtemplate_notification_templates_for_success__in=[self]))
        return dict(error=list(error_notification_templates),
                    started=list(started_notification_templates),
                    success=list(success_notification_templates))

    def _accept_or_ignore_job_kwargs(self, _exclude_errors=None, **kwargs):
        extra_data = kwargs.pop('extra_vars', {})
        prompted_data, rejected_data, errors = super(SystemJobTemplate, self)._accept_or_ignore_job_kwargs(**kwargs)
        prompted_vars, rejected_vars, errors = self.accept_or_ignore_variables(extra_data, errors, _exclude_errors=_exclude_errors)
        if prompted_vars:
            prompted_data['extra_vars'] = prompted_vars
        if rejected_vars:
            rejected_data['extra_vars'] = rejected_vars
        return (prompted_data, rejected_data, errors)

    def _accept_or_ignore_variables(self, data, errors, _exclude_errors=()):
        '''
        Unlike other templates, like project updates and inventory sources,
        system job templates can accept a limited number of fields
        used as options for the management commands.
        '''
        rejected = {}
        allowed_vars = set(['days', 'older_than', 'granularity'])
        given_vars = set(data.keys())
        unallowed_vars = given_vars - (allowed_vars & given_vars)
        errors_list = []
        if unallowed_vars:
            errors_list.append(_('Variables {list_of_keys} are not allowed for system jobs.').format(
                list_of_keys=', '.join(unallowed_vars)))
            for key in unallowed_vars:
                rejected[key] = data.pop(key)

        if 'days' in data:
            try:
                if type(data['days']) is bool:
                    raise ValueError
                if float(data['days']) != int(data['days']):
                    raise ValueError
                days = int(data['days'])
                if days < 0:
                    raise ValueError
            except ValueError:
                errors_list.append(_("days must be a positive integer."))
                rejected['days'] = data.pop('days')

        if errors_list:
            errors['extra_vars'] = errors_list
        return (data, rejected, errors)


class SystemJob(UnifiedJob, SystemJobOptions, JobNotificationMixin):

    class Meta:
        app_label = 'main'
        ordering = ('id',)

    system_job_template = models.ForeignKey(
        'SystemJobTemplate',
        related_name='jobs',
        blank=True,
        null=True,
        default=None,
        on_delete=models.SET_NULL,
    )

    extra_vars = prevent_search(models.TextField(
        blank=True,
        default='',
    ))

    extra_vars_dict = VarsDictProperty('extra_vars', True)

    @classmethod
    def _get_parent_field_name(cls):
        return 'system_job_template'

    @classmethod
    def _get_task_class(cls):
        from awx.main.tasks import RunSystemJob
        return RunSystemJob

    def websocket_emit_data(self):
        return {}

    def get_absolute_url(self, request=None):
        return reverse('api:system_job_detail', kwargs={'pk': self.pk}, request=request)

    def get_ui_url(self):
        return urljoin(settings.TOWER_URL_BASE, "/#/jobs/system/{}".format(self.pk))

    @property
    def event_class(self):
        return SystemJobEvent

    @property
    def task_impact(self):
        return 5

    @property
    def preferred_instance_groups(self):
        return self.global_instance_groups

    '''
    JobNotificationMixin
    '''
    def get_notification_templates(self):
        return self.system_job_template.notification_templates

    def get_notification_friendly_name(self):
        return "System Job"