From f5d7d0bce59822099b914f3749ed06429c903e88 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 4 Apr 2017 16:32:30 -0400 Subject: [PATCH 1/4] add model fact recent * Copy of the most recent system tracking fact for each module type. Utimately, this allows us to GIN index the jsonb object to support fact searching. --- awx/main/fields.py | 17 ++++++++ awx/main/migrations/0037_v320_fact_recent.py | 43 ++++++++++++++++++++ awx/main/models/fact.py | 38 ++++++++++++++++- 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0037_v320_fact_recent.py diff --git a/awx/main/fields.py b/awx/main/fields.py index f654c51ae4..287866b092 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -5,6 +5,7 @@ import json import re import sys +import six from pyparsing import infixNotation, opAssoc, Optional, Literal, CharsNotIn # Django @@ -45,6 +46,22 @@ class JSONField(upstream_JSONField): return {} return super(JSONField, self).from_db_value(value, expression, connection, context) +class JSONBField(upstream_JSONField): + def get_db_prep_value(self, value, connection, prepared=False): + if connection.vendor == 'sqlite': + # sqlite (which we use for tests) does not support jsonb; + return json.dumps(value) + return super(JSONBField, self).get_db_prep_value( + value, connection, prepared + ) + + def from_db_value(self, value, expression, connection, context): + # Work around a bug in django-jsonfield + # https://bitbucket.org/schinckel/django-jsonfield/issues/57/cannot-use-in-the-same-project-as-djangos + if isinstance(value, six.string_types): + return json.loads(value) + return value + # Based on AutoOneToOneField from django-annoying: # https://bitbucket.org/offline/django-annoying/src/a0de8b294db3/annoying/fields.py diff --git a/awx/main/migrations/0037_v320_fact_recent.py b/awx/main/migrations/0037_v320_fact_recent.py new file mode 100644 index 0000000000..abc22a5809 --- /dev/null +++ b/awx/main/migrations/0037_v320_fact_recent.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# Python +from __future__ import unicode_literals + +# Django +from django.db import migrations, models + +from psycopg2.extensions import AsIs + +# AWX +import awx.main.fields +from awx.main.models import FactRecent + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0036_v311_insights'), + ] + + operations = [ + migrations.CreateModel( + name='FactRecent', + fields=[ + ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), + ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)), + ('module', models.CharField(max_length=128)), + ('facts', awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)), + ('host', models.ForeignKey(related_name='facts_recent', to='main.Host', help_text='Host for the facts that the fact scan captured.')), + ], + ), + migrations.AlterField( + model_name='fact', + name='facts', + field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True), + ), + migrations.AlterIndexTogether( + name='factrecent', + index_together=set([('timestamp', 'module', 'host')]), + ), + migrations.RunSQL([("CREATE INDEX fact_recent_facts_default_gin ON %s USING gin" + "(facts jsonb_path_ops);", [AsIs(FactRecent._meta.db_table)])]), + ] diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index 480834c2c1..6881267cd8 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -6,7 +6,39 @@ from django.utils.translation import ugettext_lazy as _ from jsonbfield.fields import JSONField -__all__ = ('Fact', ) +from awx.main.fields import JSONBField + +__all__ = ('Fact', 'FactRecent') + +class FactRecent(models.Model): + host = models.ForeignKey( + 'Host', + related_name='facts_recent', + db_index=True, + on_delete=models.CASCADE, + help_text=_('Host for the facts that the fact scan captured.'), + ) + timestamp = models.DateTimeField( + default=None, + editable=False, + help_text=_('Date and time of the corresponding fact scan gathering time.') + ) + module = models.CharField(max_length=128) + facts = JSONBField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) + + class Meta: + app_label = 'main' + index_together = [ + ["timestamp", "module", "host"], + ] + + @staticmethod + def add_fact(host_id, module, timestamp, facts): + qs = FactRecent.objects.filter(host_id=host_id, module=module) + qs.delete() + + fact_obj = FactRecent.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + return fact_obj class Fact(models.Model): @@ -26,7 +58,7 @@ class Fact(models.Model): help_text=_('Date and time of the corresponding fact scan gathering time.') ) module = models.CharField(max_length=128) - facts = JSONField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) + facts = JSONBField(blank=True, default={}, help_text=_('Arbitrary JSON structure of module facts captured at timestamp for a single host.')) class Meta: app_label = 'main' @@ -60,6 +92,8 @@ class Fact(models.Model): @staticmethod def add_fact(host_id, module, timestamp, facts): + FactRecent.add_fact(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + fact_obj = Fact.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj.save() return fact_obj From f7fb541fe247ebc7ce791c81dd7580c7cef7edc0 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Tue, 4 Apr 2017 17:09:07 -0400 Subject: [PATCH 2/4] add facts recent endpoints * Hang most recent listing of facts per-host off of /hosts/n/ --- awx/api/serializers.py | 1 + awx/api/urls.py | 1 + awx/api/views.py | 9 +++++++++ awx/main/access.py | 6 ++++++ awx/main/fields.py | 1 + awx/main/models/fact.py | 4 ++-- 6 files changed, 20 insertions(+), 2 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 6dc20e1215..51ac883ee8 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1157,6 +1157,7 @@ class HostSerializer(BaseSerializerWithVariables): ad_hoc_commands = self.reverse('api:host_ad_hoc_commands_list', kwargs={'pk': obj.pk}), ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}), fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}), + facts_recent = self.reverse('api:host_facts_recent_list', kwargs={'pk': obj.pk}), )) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) diff --git a/awx/api/urls.py b/awx/api/urls.py index 9e5e0f5895..64a2749d01 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -114,6 +114,7 @@ host_urls = patterns('awx.api.views', #url(r'^(?P[0-9]+)/single_fact/$', 'host_single_fact_view'), url(r'^(?P[0-9]+)/fact_versions/$', 'host_fact_versions_list'), url(r'^(?P[0-9]+)/fact_view/$', 'host_fact_compare_view'), + url(r'^(?P[0-9]+)/facts_recent/$', 'host_facts_recent_list'), ) group_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index da49187df7..edb032874a 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1835,6 +1835,15 @@ class HostFactCompareView(SystemTrackingEnforcementMixin, SubDetailAPIView): return Response(self.serializer_class(instance=fact_entry).data) +class HostFactsRecentList(SubListAPIView): + + model = FactRecent + parent_model = Host + relationship = 'facts_recent' + serializer_class = FactSerializer + new_in_320 = True + + class GroupList(ListCreateAPIView): model = Group diff --git a/awx/main/access.py b/awx/main/access.py index 20becf2cbe..151b00b20c 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2264,7 +2264,12 @@ class RoleAccess(BaseAccess): return False +class FactRecentAccess(BaseAccess): + model = FactRecent + + def get_queryset(self): + return FactRecent.objects.distinct() register_access(User, UserAccess) @@ -2299,3 +2304,4 @@ register_access(WorkflowJobTemplateNode, WorkflowJobTemplateNodeAccess) register_access(WorkflowJobNode, WorkflowJobNodeAccess) register_access(WorkflowJobTemplate, WorkflowJobTemplateAccess) register_access(WorkflowJob, WorkflowJobAccess) +register_access(FactRecent, FactRecentAccess) diff --git a/awx/main/fields.py b/awx/main/fields.py index 287866b092..373133eb60 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -46,6 +46,7 @@ class JSONField(upstream_JSONField): return {} return super(JSONField, self).from_db_value(value, expression, connection, context) + class JSONBField(upstream_JSONField): def get_db_prep_value(self, value, connection, prepared=False): if connection.vendor == 'sqlite': diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index 6881267cd8..e07f37448a 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -4,12 +4,11 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ -from jsonbfield.fields import JSONField - from awx.main.fields import JSONBField __all__ = ('Fact', 'FactRecent') + class FactRecent(models.Model): host = models.ForeignKey( 'Host', @@ -97,3 +96,4 @@ class Fact(models.Model): fact_obj = Fact.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj.save() return fact_obj + From 02795e526cea2186871d4ccbf4dd302f6841c33c Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Apr 2017 11:11:46 -0400 Subject: [PATCH 3/4] inherit from jsonbfield package not jsonfield * jsonbfield supports json querying. jsonfield package is still a jsonb postgres data type, but doesn't support jsonb style querying. * add undo migration support to GIN index --- awx/main/fields.py | 9 ++++++--- awx/main/migrations/0037_v320_fact_recent.py | 3 ++- awx/main/tests/unit/test_fields.py | 3 ++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/awx/main/fields.py b/awx/main/fields.py index 373133eb60..aeb5101dc9 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -27,6 +27,7 @@ from django.db.models import Q # Django-JSONField from jsonfield import JSONField as upstream_JSONField +from jsonbfield.fields import JSONField as upstream_JSONBField # AWX from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role @@ -47,7 +48,7 @@ class JSONField(upstream_JSONField): return super(JSONField, self).from_db_value(value, expression, connection, context) -class JSONBField(upstream_JSONField): +class JSONBField(upstream_JSONBField): def get_db_prep_value(self, value, connection, prepared=False): if connection.vendor == 'sqlite': # sqlite (which we use for tests) does not support jsonb; @@ -399,8 +400,10 @@ class DynamicFilterField(models.TextField): last_v = new_v last_kv = new_kv contains_count += 1 - - if contains_count > 1: + + if contains_count == 1 and isinstance(assembled_v, basestring): + assembled_v = '"' + assembled_v + '"' + elif contains_count > 1: if type(last_v) is list: last_v.append(v) if type(last_v) is dict: diff --git a/awx/main/migrations/0037_v320_fact_recent.py b/awx/main/migrations/0037_v320_fact_recent.py index abc22a5809..9a8ffb58b8 100644 --- a/awx/main/migrations/0037_v320_fact_recent.py +++ b/awx/main/migrations/0037_v320_fact_recent.py @@ -39,5 +39,6 @@ class Migration(migrations.Migration): index_together=set([('timestamp', 'module', 'host')]), ), migrations.RunSQL([("CREATE INDEX fact_recent_facts_default_gin ON %s USING gin" - "(facts jsonb_path_ops);", [AsIs(FactRecent._meta.db_table)])]), + "(facts jsonb_path_ops);", [AsIs(FactRecent._meta.db_table)])], + [('DROP INDEX fact_recent_facts_default_gin;', None)]), ] diff --git a/awx/main/tests/unit/test_fields.py b/awx/main/tests/unit/test_fields.py index 468a90da9c..f203d42d4d 100644 --- a/awx/main/tests/unit/test_fields.py +++ b/awx/main/tests/unit/test_fields.py @@ -59,7 +59,8 @@ class TestDynamicFilterFieldFilterStringToQ(): ('a__b__c[]=3.14', Q(**{ "a__b__c__contains": 3.14})), ('a__b__c[]=true', Q(**{ "a__b__c__contains": True})), ('a__b__c[]=false', Q(**{ "a__b__c__contains": False})), - ('a__b__c[]="true"', Q(**{ "a__b__c__contains": "true"})), + ('a__b__c[]="true"', Q(**{ "a__b__c__contains": "\"true\""})), + ('a__b__c[]="hello world"', Q(**{ "a__b__c__contains": "\"hello world\""})), ('a__b__c[]__d[]="foobar"', Q(**{ "a__b__c__contains": [{"d": ["foobar"]}]})), ('a__b__c[]__d="foobar"', Q(**{ "a__b__c__contains": [{"d": "foobar"}]})), ('a__b__c[]__d__e="foobar"', Q(**{ "a__b__c__contains": [{"d": {"e": "foobar"}}]})), From ee09bca5586ea6d79098af92cb288d57c41ef01e Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Wed, 5 Apr 2017 16:50:38 -0400 Subject: [PATCH 4/4] change facts_recent to facts_latest --- awx/api/serializers.py | 2 +- awx/api/urls.py | 2 +- awx/api/views.py | 6 +++--- awx/main/access.py | 8 ++++---- awx/main/migrations/0037_v320_fact_recent.py | 14 +++++++------- awx/main/models/fact.py | 12 ++++++------ 6 files changed, 22 insertions(+), 22 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 51ac883ee8..d954f532af 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -1157,7 +1157,7 @@ class HostSerializer(BaseSerializerWithVariables): ad_hoc_commands = self.reverse('api:host_ad_hoc_commands_list', kwargs={'pk': obj.pk}), ad_hoc_command_events = self.reverse('api:host_ad_hoc_command_events_list', kwargs={'pk': obj.pk}), fact_versions = self.reverse('api:host_fact_versions_list', kwargs={'pk': obj.pk}), - facts_recent = self.reverse('api:host_facts_recent_list', kwargs={'pk': obj.pk}), + facts_latest = self.reverse('api:host_facts_latest_list', kwargs={'pk': obj.pk}), )) if obj.inventory: res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk}) diff --git a/awx/api/urls.py b/awx/api/urls.py index 64a2749d01..973075a651 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -114,7 +114,7 @@ host_urls = patterns('awx.api.views', #url(r'^(?P[0-9]+)/single_fact/$', 'host_single_fact_view'), url(r'^(?P[0-9]+)/fact_versions/$', 'host_fact_versions_list'), url(r'^(?P[0-9]+)/fact_view/$', 'host_fact_compare_view'), - url(r'^(?P[0-9]+)/facts_recent/$', 'host_facts_recent_list'), + url(r'^(?P[0-9]+)/facts_latest/$', 'host_facts_latest_list'), ) group_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index edb032874a..9f6f2d5b1a 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -1835,11 +1835,11 @@ class HostFactCompareView(SystemTrackingEnforcementMixin, SubDetailAPIView): return Response(self.serializer_class(instance=fact_entry).data) -class HostFactsRecentList(SubListAPIView): +class HostFactsLatestList(SubListAPIView): - model = FactRecent + model = FactLatest parent_model = Host - relationship = 'facts_recent' + relationship = 'facts_latest' serializer_class = FactSerializer new_in_320 = True diff --git a/awx/main/access.py b/awx/main/access.py index 151b00b20c..f4c4486faf 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -2264,12 +2264,12 @@ class RoleAccess(BaseAccess): return False -class FactRecentAccess(BaseAccess): +class FactLatestAccess(BaseAccess): - model = FactRecent + model = FactLatest def get_queryset(self): - return FactRecent.objects.distinct() + return FactLatest.objects.distinct() register_access(User, UserAccess) @@ -2304,4 +2304,4 @@ register_access(WorkflowJobTemplateNode, WorkflowJobTemplateNodeAccess) register_access(WorkflowJobNode, WorkflowJobNodeAccess) register_access(WorkflowJobTemplate, WorkflowJobTemplateAccess) register_access(WorkflowJob, WorkflowJobAccess) -register_access(FactRecent, FactRecentAccess) +register_access(FactLatest, FactLatestAccess) diff --git a/awx/main/migrations/0037_v320_fact_recent.py b/awx/main/migrations/0037_v320_fact_recent.py index 9a8ffb58b8..d7ca92f154 100644 --- a/awx/main/migrations/0037_v320_fact_recent.py +++ b/awx/main/migrations/0037_v320_fact_recent.py @@ -9,7 +9,7 @@ from psycopg2.extensions import AsIs # AWX import awx.main.fields -from awx.main.models import FactRecent +from awx.main.models import FactLatest class Migration(migrations.Migration): @@ -20,13 +20,13 @@ class Migration(migrations.Migration): operations = [ migrations.CreateModel( - name='FactRecent', + name='FactLatest', fields=[ ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), ('timestamp', models.DateTimeField(default=None, help_text='Date and time of the corresponding fact scan gathering time.', editable=False)), ('module', models.CharField(max_length=128)), ('facts', awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True)), - ('host', models.ForeignKey(related_name='facts_recent', to='main.Host', help_text='Host for the facts that the fact scan captured.')), + ('host', models.ForeignKey(related_name='facts_latest', to='main.Host', help_text='Host for the facts that the fact scan captured.')), ], ), migrations.AlterField( @@ -35,10 +35,10 @@ class Migration(migrations.Migration): field=awx.main.fields.JSONBField(default={}, help_text='Arbitrary JSON structure of module facts captured at timestamp for a single host.', blank=True), ), migrations.AlterIndexTogether( - name='factrecent', + name='factlatest', index_together=set([('timestamp', 'module', 'host')]), ), - migrations.RunSQL([("CREATE INDEX fact_recent_facts_default_gin ON %s USING gin" - "(facts jsonb_path_ops);", [AsIs(FactRecent._meta.db_table)])], - [('DROP INDEX fact_recent_facts_default_gin;', None)]), + migrations.RunSQL([("CREATE INDEX fact_latest_facts_default_gin ON %s USING gin" + "(facts jsonb_path_ops);", [AsIs(FactLatest._meta.db_table)])], + [('DROP INDEX fact_latest_facts_default_gin;', None)]), ] diff --git a/awx/main/models/fact.py b/awx/main/models/fact.py index e07f37448a..3c4ffabc80 100644 --- a/awx/main/models/fact.py +++ b/awx/main/models/fact.py @@ -6,13 +6,13 @@ from django.utils.translation import ugettext_lazy as _ from awx.main.fields import JSONBField -__all__ = ('Fact', 'FactRecent') +__all__ = ('Fact', 'FactLatest') -class FactRecent(models.Model): +class FactLatest(models.Model): host = models.ForeignKey( 'Host', - related_name='facts_recent', + related_name='facts_latest', db_index=True, on_delete=models.CASCADE, help_text=_('Host for the facts that the fact scan captured.'), @@ -33,10 +33,10 @@ class FactRecent(models.Model): @staticmethod def add_fact(host_id, module, timestamp, facts): - qs = FactRecent.objects.filter(host_id=host_id, module=module) + qs = FactLatest.objects.filter(host_id=host_id, module=module) qs.delete() - fact_obj = FactRecent.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + fact_obj = FactLatest.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) return fact_obj @@ -91,7 +91,7 @@ class Fact(models.Model): @staticmethod def add_fact(host_id, module, timestamp, facts): - FactRecent.add_fact(host_id=host_id, module=module, timestamp=timestamp, facts=facts) + FactLatest.add_fact(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj = Fact.objects.create(host_id=host_id, module=module, timestamp=timestamp, facts=facts) fact_obj.save()