From 55268a8ff01ac2a600d7fd3f6b3c34d097e07a55 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 17 Apr 2017 14:27:33 -0400 Subject: [PATCH] remove facts_latest in favor of ansible_facts * Closer align our facts data structure with ansible facts data structure for purposes of ... wait for it ... 2-way fact caching --- awx/api/serializers.py | 10 +++++- awx/api/urls.py | 2 +- awx/api/views.py | 16 ++++----- awx/main/access.py | 9 ----- awx/main/migrations/0037_v320_release.py | 32 +++++++---------- awx/main/models/fact.py | 44 +++++++----------------- awx/main/models/inventory.py | 17 ++++++++- 7 files changed, 57 insertions(+), 73 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 9297a6f117..c9db59a166 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1157,7 +1157,7 @@ class HostSerializer(BaseSerializerWithVariables): ad_hoc_commands = self.reverse('api:host_ad_hoc_commands_list', kwargs={'pk': obj.pk}), ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}), fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}), - facts_latest = self.reverse('api:host_facts_latest_list', kwargs={'pk': obj.pk}), + ansible_facts = self.reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.pk}), )) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) @@ -1237,6 +1237,14 @@ class HostSerializer(BaseSerializerWithVariables): return ret +class AnsibleFactsSerializer(BaseSerializer): + class Meta: + model = Host + + def to_representation(self, obj): + return obj.ansible_facts + + class GroupSerializer(BaseSerializerWithVariables): inventory_source = serializers.SerializerMethodField( help_text=_('Dedicated inventory source for the group, will be removed in 3.3.')) diff --git a/awx/api/urls.py b/awx/api/urls.py index 973075a651..daa02d4329 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -114,7 +114,7 @@ host_urls = patterns('awx.api.views', #url(r'^(?P[0-9]+)/single_fact/$', 'host_single_fact_view'), url(r'^(?P[0-9]+)/fact_versions/$', 'host_fact_versions_list'), url(r'^(?P[0-9]+)/fact_view/$', 'host_fact_compare_view'), - url(r'^(?P[0-9]+)/facts_latest/$', 'host_facts_latest_list'), + url(r'^(?P[0-9]+)/ansible_facts/$', 'host_ansible_facts_detail'), ) group_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index f70017b7ce..784f5a13bd 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1725,6 +1725,13 @@ class HostDetail(RetrieveUpdateDestroyAPIView): serializer_class = HostSerializer +class HostAnsibleFactsDetail(RetrieveAPIView): + + model = Host + serializer_class = AnsibleFactsSerializer + new_in_320 = True + + class InventoryHostsList(SubListCreateAttachDetachAPIView): model = Host @@ -1842,15 +1849,6 @@ class HostFactCompareView(SystemTrackingEnforcementMixin, SubDetailAPIView): return Response(self.serializer_class(instance=fact_entry).data) -class HostFactsLatestList(SubListAPIView): - - model = FactLatest - parent_model = Host - relationship = 'facts_latest' - serializer_class = FactSerializer - new_in_320 = True - - class GroupList(ListCreateAPIView): model = Group diff --git a/awx/main/access.py b/awx/main/access.py index cac7ac88f1..0d363ea3c7 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2274,14 +2274,6 @@ class RoleAccess(BaseAccess): return False -class FactLatestAccess(BaseAccess): - - model = FactLatest - - def get_queryset(self): - return FactLatest.objects.distinct() - - register_access(User, UserAccess) register_access(Organization, OrganizationAccess) register_access(Inventory, InventoryAccess) @@ -2314,4 +2306,3 @@ register_access(WorkflowJobTemplateNode, WorkflowJobTemplateNodeAccess) register_access(WorkflowJobNode, WorkflowJobNodeAccess) register_access(WorkflowJobTemplate, WorkflowJobTemplateAccess) register_access(WorkflowJob, WorkflowJobAccess) -register_access(FactLatest, FactLatestAccess) diff --git a/awx/main/migrations/0037_v320_release.py b/awx/main/migrations/0037_v320_release.py index 37a3a0795e..6049a9d385 100644 --- a/awx/main/migrations/0037_v320_release.py +++ b/awx/main/migrations/0037_v320_release.py @@ -2,14 +2,15 @@ # Python from __future__ import unicode_literals +# Psycopg2 +from psycopg2.extensions import AsIs + # Django from django.db import migrations, models -from psycopg2.extensions import AsIs - # AWX import awx.main.fields -from awx.main.models import FactLatest +from awx.main.models import Host class Migration(migrations.Migration): @@ -36,27 +37,18 @@ class Migration(migrations.Migration): field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True), ), - # Facts Latest - migrations.CreateModel( - name='FactLatest', - fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)), - ('module', models.CharField(max_length=128)), - ('facts', awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)), - ('host', models.ForeignKey(related_name='facts_latest', to='main.Host', help_text='Host for the facts that the fact scan captured.')), - ], - ), + # Facts migrations.AlterField( model_name='fact', name='facts', field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True), ), - migrations.AlterIndexTogether( - name='factlatest', - index_together=set([('timestamp', 'module', 'host')]), + migrations.AddField( + model_name='host', + name='ansible_facts', + field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of most recent ansible_facts, per-host.', blank=True), ), - migrations.RunSQL([("CREATE INDEX fact_latest_facts_default_gin ON %s USING gin" - "(facts jsonb_path_ops);", [AsIs(FactLatest._meta.db_table)])], - [('DROP INDEX fact_latest_facts_default_gin;', None)]), + migrations.RunSQL([("CREATE INDEX host_ansible_facts_default_gin ON %s USING gin" + "(ansible_facts jsonb_path_ops);", [AsIs(Host._meta.db_table)])], + [('DROP INDEX host_ansible_facts_default_gin;', None)]), ] diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index 3c4ffabc80..28ba767e3e 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -1,43 +1,17 @@ # Copyright (c) 2016 Ansible, Inc. # All Rights Reserved. +import logging + from django.db import models from django.utils.translation import ugettext_lazy as _ from awx.main.fields import JSONBField +from awx.main.models import Host -__all__ = ('Fact', 'FactLatest') +__all__ = ('Fact',) - -class FactLatest(models.Model): - host = models.ForeignKey( - 'Host', - related_name='facts_latest', - db_index=True, - on_delete=models.CASCADE, - help_text=_('Host for the facts that the fact scan captured.'), - ) - timestamp = models.DateTimeField( - default=None, - editable=False, - help_text=_('Date and time of the corresponding fact scan gathering time.') - ) - module = models.CharField(max_length=128) - facts = JSONBField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) - - class Meta: - app_label = 'main' - index_together = [ - ["timestamp", "module", "host"], - ] - - @staticmethod - def add_fact(host_id, module, timestamp, facts): - qs = FactLatest.objects.filter(host_id=host_id, module=module) - qs.delete() - - fact_obj = FactLatest.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) - return fact_obj +logger = logging.getLogger('awx.main.models.fact') class Fact(models.Model): @@ -91,7 +65,13 @@ class Fact(models.Model): @staticmethod def add_fact(host_id, module, timestamp, facts): - FactLatest.add_fact(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + try: + host = Host.objects.get(id=host_id) + except Host.DoesNotExist as e: + logger.warn("Host with id %s not found while trying to update latest fact set." % host_id) + raise e + + host.update_ansible_facts(module=module, facts=facts, timestamp=timestamp) fact_obj = Fact.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj.save() diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 1d4792fa7a..789804e8b5 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -19,7 +19,7 @@ from django.utils.timezone import now # AWX from awx.api.versioning import reverse from awx.main.constants import CLOUD_PROVIDERS -from awx.main.fields import ImplicitRoleField +from awx.main.fields import ImplicitRoleField, JSONBField from awx.main.managers import HostManager from awx.main.models.base import * # noqa from awx.main.models.unified_jobs import * # noqa @@ -382,6 +382,11 @@ class Host(CommonModelNameNotUnique): editable=False, help_text=_('Inventory source(s) that created or modified this host.'), ) + ansible_facts = JSONBField( + blank=True, + default={}, + help_text=_('Arbitrary JSON structure of most recent ansible_facts, per-host.'), + ) objects = HostManager() @@ -444,6 +449,16 @@ class Host(CommonModelNameNotUnique): # Use .job_host_summaries.all() to get jobs affecting this host. # Use .job_events.all() to get events affecting this host. + ''' + We don't use timestamp, but we may in the future. + ''' + def update_ansible_facts(self, module, facts, timestamp=None): + if module == "ansible": + self.ansible_facts.update(facts) + else: + self.ansible_facts[module] = facts + self.save() + class Group(CommonModelNameNotUnique): '''