From f5d7d0bce59822099b914f3749ed06429c903e88 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 4 Apr 2017 16:32:30 -0400 Subject: [PATCH] add model fact recent * Copy of the most recent system tracking fact for each module type. Utimately, this allows us to GIN index the jsonb object to support fact searching. --- awx/main/fields.py | 17 ++++++++ awx/main/migrations/0037_v320_fact_recent.py | 43 ++++++++++++++++++++ awx/main/models/fact.py | 38 ++++++++++++++++- 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0037_v320_fact_recent.py diff --git a/awx/main/fields.py b/awx/main/fields.py index f654c51ae4..287866b092 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -5,6 +5,7 @@ import json import re import sys +import six from pyparsing import infixNotation, opAssoc, Optional, Literal, CharsNotIn # Django @@ -45,6 +46,22 @@ class JSONField(upstream_JSONField): return {} return super(JSONField, self).from_db_value(value, expression, connection, context) +class JSONBField(upstream_JSONField): + def get_db_prep_value(self, value, connection, prepared=False): + if connection.vendor == 'sqlite': + # sqlite (which we use for tests) does not support jsonb; + return json.dumps(value) + return super(JSONBField, self).get_db_prep_value( + value, connection, prepared + ) + + def from_db_value(self, value, expression, connection, context): + # Work around a bug in django-jsonfield + # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos + if isinstance(value, six.string_types): + return json.loads(value) + return value + # Based on AutoOneToOneField from django-annoying: # https://bitbucket.org/offline/django-annoying/src/a0de8b294db3/annoying/fields.py diff --git a/awx/main/migrations/0037_v320_fact_recent.py b/awx/main/migrations/0037_v320_fact_recent.py new file mode 100644 index 0000000000..abc22a5809 --- /dev/null +++ b/awx/main/migrations/0037_v320_fact_recent.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Python +from __future__ import unicode_literals + +# Django +from django.db import migrations, models + +from psycopg2.extensions import AsIs + +# AWX +import awx.main.fields +from awx.main.models import FactRecent + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0036_v311_insights'), + ] + + operations = [ + migrations.CreateModel( + name='FactRecent', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)), + ('module', models.CharField(max_length=128)), + ('facts', awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)), + ('host', models.ForeignKey(related_name='facts_recent', to='main.Host', help_text='Host for the facts that the fact scan captured.')), + ], + ), + migrations.AlterField( + model_name='fact', + name='facts', + field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True), + ), + migrations.AlterIndexTogether( + name='factrecent', + index_together=set([('timestamp', 'module', 'host')]), + ), + migrations.RunSQL([("CREATE INDEX fact_recent_facts_default_gin ON %s USING gin" + "(facts jsonb_path_ops);", [AsIs(FactRecent._meta.db_table)])]), + ] diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index 480834c2c1..6881267cd8 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -6,7 +6,39 @@ from django.utils.translation import ugettext_lazy as _ from jsonbfield.fields import JSONField -__all__ = ('Fact', ) +from awx.main.fields import JSONBField + +__all__ = ('Fact', 'FactRecent') + +class FactRecent(models.Model): + host = models.ForeignKey( + 'Host', + related_name='facts_recent', + db_index=True, + on_delete=models.CASCADE, + help_text=_('Host for the facts that the fact scan captured.'), + ) + timestamp = models.DateTimeField( + default=None, + editable=False, + help_text=_('Date and time of the corresponding fact scan gathering time.') + ) + module = models.CharField(max_length=128) + facts = JSONBField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) + + class Meta: + app_label = 'main' + index_together = [ + ["timestamp", "module", "host"], + ] + + @staticmethod + def add_fact(host_id, module, timestamp, facts): + qs = FactRecent.objects.filter(host_id=host_id, module=module) + qs.delete() + + fact_obj = FactRecent.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + return fact_obj class Fact(models.Model): @@ -26,7 +58,7 @@ class Fact(models.Model): help_text=_('Date and time of the corresponding fact scan gathering time.') ) module = models.CharField(max_length=128) - facts = JSONField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) + facts = JSONBField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) class Meta: app_label = 'main' @@ -60,6 +92,8 @@ class Fact(models.Model): @staticmethod def add_fact(host_id, module, timestamp, facts): + FactRecent.add_fact(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + fact_obj = Fact.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj.save() return fact_obj