From d652ed16d056ad409de4dcd71a48eccd6bbbe91f Mon Sep 17 00:00:00 2001 From: Wayne Witzel III Date: Wed, 17 May 2017 16:25:40 -0400 Subject: [PATCH] Dynamic -> Smart Inventory --- awx/api/serializers.py | 8 +-- awx/api/views.py | 6 +-- awx/main/fields.py | 10 ++-- awx/main/managers.py | 8 +-- awx/main/migrations/0038_v320_release.py | 6 +-- awx/main/models/inventory.py | 12 ++--- .../tests/functional/models/test_inventory.py | 52 +++++++++---------- awx/main/tests/unit/utils/test_filters.py | 18 +++---- awx/main/utils/filters.py | 14 ++--- 9 files changed, 67 insertions(+), 67 deletions(-) diff --git a/awx/api/serializers.py b/awx/api/serializers.py index c4e7b08867..72b33057fe 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -45,7 +45,7 @@ from awx.main.fields import ImplicitRoleField from awx.main.utils import ( get_type_for_model, get_model_for_type, timestamp_apiformat, camelcase_to_underscore, getattrd, parse_yaml_or_json) -from awx.main.utils.filters import DynamicFilter +from awx.main.utils.filters import SmartFilter from awx.main.validators import vars_validate_or_raise @@ -1144,11 +1144,11 @@ class InventorySerializer(BaseSerializerWithVariables): def validate(self, attrs): kind = attrs.get('kind', 'standard') - if kind == 'dynamic': + if kind == 'smart': host_filter = attrs.get('host_filter') if host_filter is not None: try: - DynamicFilter().query_from_string(host_filter) + SmartFilter().query_from_string(host_filter) except RuntimeError, e: raise models.base.ValidationError(e) return super(InventorySerializer, self).validate(attrs) @@ -3437,7 +3437,7 @@ class InstanceGroupSerializer(BaseSerializer): def get_instances(self, obj): return obj.instances.count() - + class ActivityStreamSerializer(BaseSerializer): changes = serializers.SerializerMethodField() diff --git a/awx/api/views.py b/awx/api/views.py index ffe53d1fb9..8116f35bbb 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -72,7 +72,7 @@ from awx.main.utils import * # noqa from awx.main.utils import ( callback_filter_out_ansible_extra_vars ) -from awx.main.utils.filters import DynamicFilter +from awx.main.utils.filters import SmartFilter from awx.api.permissions import * # noqa from awx.api.renderers import * # noqa @@ -554,7 +554,7 @@ class InstanceGroupDetail(RetrieveAPIView): serializer_class = InstanceGroupSerializer new_in_320 = True - + class InstanceGroupUnifiedJobsList(SubListAPIView): view_name = _("Instance Group Running Jobs") @@ -1924,7 +1924,7 @@ class HostList(ListCreateAPIView): qs = super(HostList, self).get_queryset() filter_string = self.request.query_params.get('host_filter', None) if filter_string: - filter_qs = DynamicFilter.query_from_string(filter_string) + filter_qs = SmartFilter.query_from_string(filter_string) qs &= filter_qs return qs diff --git a/awx/main/fields.py b/awx/main/fields.py index 344a3bc1c7..89e1aae75d 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -36,13 +36,13 @@ from jsonfield import JSONField as upstream_JSONField from jsonbfield.fields import JSONField as upstream_JSONBField # AWX -from awx.main.utils.filters import DynamicFilter +from awx.main.utils.filters import SmartFilter from awx.main.validators import validate_ssh_private_key from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role from awx.main import utils -__all__ = ['AutoOneToOneField', 'ImplicitRoleField', 'JSONField', 'DynamicFilterField'] +__all__ = ['AutoOneToOneField', 'ImplicitRoleField', 'JSONField', 'SmartFilterField'] class JSONField(upstream_JSONField): @@ -331,17 +331,17 @@ class ImplicitRoleField(models.ForeignKey): Role.rebuild_role_ancestor_list([], child_ids) -class DynamicFilterField(models.TextField): +class SmartFilterField(models.TextField): def get_prep_value(self, value): # Change any false value to none. # https://docs.python.org/2/library/stdtypes.html#truth-value-testing if not value: return None try: - DynamicFilter().query_from_string(value) + SmartFilter().query_from_string(value) except RuntimeError, e: raise models.base.ValidationError(e) - return super(DynamicFilterField, self).get_prep_value(value) + return super(SmartFilterField, self).get_prep_value(value) class JSONSchemaField(JSONBField): diff --git a/awx/main/managers.py b/awx/main/managers.py index 0ce2068a14..2deeeb7046 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -9,7 +9,7 @@ from django.utils.timezone import now from django.db.models import Sum from django.conf import settings -from awx.main.utils.filters import DynamicFilter +from awx.main.utils.filters import SmartFilter ___all__ = ['HostManager', 'InstanceManager'] @@ -25,15 +25,15 @@ class HostManager(models.Manager): return len(set(self.values_list('name', flat=True))) def get_queryset(self): - """When the parent instance of the host query set has a `kind` of dynamic and a `host_filter` + """When the parent instance of the host query set has a `kind=smart` and a `host_filter` set. Use the `host_filter` to generate the queryset for the hosts. """ qs = super(HostManager, self).get_queryset() if (hasattr(self, 'instance') and hasattr(self.instance, 'host_filter') and hasattr(self.instance, 'kind')): - if self.instance.kind == 'dynamic' and self.instance.host_filter is not None: - q = DynamicFilter.query_from_string(self.instance.host_filter) + if self.instance.kind == 'smart' and self.instance.host_filter is not None: + q = SmartFilter.query_from_string(self.instance.host_filter) # If we are using host_filters, disable the core_filters, this allows # us to access all of the available Host entries, not just the ones associated # with a specific FK/relation. diff --git a/awx/main/migrations/0038_v320_release.py b/awx/main/migrations/0038_v320_release.py index d28a8cce0b..ec8c0e4a8e 100644 --- a/awx/main/migrations/0038_v320_release.py +++ b/awx/main/migrations/0038_v320_release.py @@ -39,12 +39,12 @@ class Migration(migrations.Migration): migrations.AddField( model_name='inventory', name='host_filter', - field=awx.main.fields.DynamicFilterField(default=None, help_text='Filter that will be applied to the hosts of this inventory.', null=True, blank=True), + field=awx.main.fields.SmartFilterField(default=None, help_text='Filter that will be applied to the hosts of this inventory.', null=True, blank=True), ), migrations.AddField( model_name='inventory', name='kind', - field=models.CharField(default=b'standard', help_text='Kind of inventory being represented.', max_length=32, choices=[(b'standard', 'Hosts have a direct link to this inventory.'), (b'dynamic', 'Hosts for inventory generated using the host_filter property.')]), + field=models.CharField(default=b'', help_text='Kind of inventory being represented.', max_length=32, choices=[(b'', 'Hosts have a direct link to this inventory.'), (b'smart', 'Hosts for inventory generated using the host_filter property.')]), ), # Facts @@ -236,7 +236,7 @@ class Migration(migrations.Migration): migrations.DeleteModel( name='Permission', ), - + # Insights migrations.AddField( model_name='host', diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 129dbef196..3326fc44b4 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -23,7 +23,7 @@ from awx.main.constants import CLOUD_PROVIDERS from awx.main.fields import ( ImplicitRoleField, JSONBField, - DynamicFilterField, + SmartFilterField, ) from awx.main.managers import HostManager from awx.main.models.base import * # noqa @@ -47,8 +47,8 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): ''' KIND_CHOICES = [ - ('standard', _('Hosts have a direct link to this inventory.')), - ('dynamic', _('Hosts for inventory generated using the host_filter property.')), + ('', _('Hosts have a direct link to this inventory.')), + ('smart', _('Hosts for inventory generated using the host_filter property.')), ] class Meta: @@ -111,11 +111,11 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin): kind = models.CharField( max_length=32, choices=KIND_CHOICES, - blank=False, - default='standard', + blank=True, + default='', help_text=_('Kind of inventory being represented.'), ) - host_filter = DynamicFilterField( + host_filter = SmartFilterField( blank=True, null=True, default=None, diff --git a/awx/main/tests/functional/models/test_inventory.py b/awx/main/tests/functional/models/test_inventory.py index dbb41eb30e..4c16a83c6f 100644 --- a/awx/main/tests/functional/models/test_inventory.py +++ b/awx/main/tests/functional/models/test_inventory.py @@ -83,39 +83,39 @@ def setup_inventory_groups(inventory, group_factory): @pytest.mark.django_db class TestHostManager: def test_host_filter_change(self, setup_ec2_gce, organization): - dynamic_inventory = Inventory(name='dynamic', - kind='dynamic', - organization=organization, - host_filter='inventory_sources__source=ec2') - dynamic_inventory.save() - assert len(dynamic_inventory.hosts.all()) == 2 + smart_inventory = Inventory(name='smart', + kind='smart', + organization=organization, + host_filter='inventory_sources__source=ec2') + smart_inventory.save() + assert len(smart_inventory.hosts.all()) == 2 - dynamic_inventory.host_filter = 'inventory_sources__source=gce' - dynamic_inventory.save() - assert len(dynamic_inventory.hosts.all()) == 1 + smart_inventory.host_filter = 'inventory_sources__source=gce' + smart_inventory.save() + assert len(smart_inventory.hosts.all()) == 1 - def test_host_filter_not_dynamic(self, setup_ec2_gce, organization): - dynamic_inventory = Inventory(name='dynamic', - organization=organization, - host_filter='inventory_sources__source=ec2') - assert len(dynamic_inventory.hosts.all()) == 0 + def test_host_filter_not_smart(self, setup_ec2_gce, organization): + smart_inventory = Inventory(name='smart', + organization=organization, + host_filter='inventory_sources__source=ec2') + assert len(smart_inventory.hosts.all()) == 0 def test_host_objects_manager(self, setup_ec2_gce, organization): - dynamic_inventory = Inventory(kind='dynamic', - name='dynamic', - organization=organization, - host_filter='inventory_sources__source=ec2') - dynamic_inventory.save() + smart_inventory = Inventory(kind='smart', + name='smart', + organization=organization, + host_filter='inventory_sources__source=ec2') + smart_inventory.save() - hosts = dynamic_inventory.hosts.all() + hosts = smart_inventory.hosts.all() assert len(hosts) == 2 assert hosts[0].inventory_sources.first().source == 'ec2' assert hosts[1].inventory_sources.first().source == 'ec2' def test_host_objects_no_dupes(self, setup_inventory_groups, organization): - dynamic_inventory = Inventory(name='dynamic', - kind='dynamic', - organization=organization, - host_filter='groups__name=test_groupA or groups__name=test_groupB') - dynamic_inventory.save() - assert len(dynamic_inventory.hosts.all()) == 1 + smart_inventory = Inventory(name='smart', + kind='smart', + organization=organization, + host_filter='groups__name=test_groupA or groups__name=test_groupB') + smart_inventory.save() + assert len(smart_inventory.hosts.all()) == 1 diff --git a/awx/main/tests/unit/utils/test_filters.py b/awx/main/tests/unit/utils/test_filters.py index b2f034d782..910217deab 100644 --- a/awx/main/tests/unit/utils/test_filters.py +++ b/awx/main/tests/unit/utils/test_filters.py @@ -4,7 +4,7 @@ import pytest import mock # AWX -from awx.main.utils.filters import DynamicFilter +from awx.main.utils.filters import SmartFilter # Django from django.db.models import Q @@ -22,7 +22,7 @@ class mockHost: @mock.patch('awx.main.utils.filters.get_host_model', return_value=mockHost()) -class TestDynamicFilterQueryFromString(): +class TestSmartFilterQueryFromString(): @pytest.mark.parametrize("filter_string,q_expected", [ ('facts__facts__blank=""', Q(**{u"facts__facts__blank": u""})), ('"facts__facts__ space "="f"', Q(**{u"facts__facts__ space ": u"f"})), @@ -36,7 +36,7 @@ class TestDynamicFilterQueryFromString(): #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})), ]) def test_query_generated(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) @pytest.mark.parametrize("filter_string", [ @@ -45,7 +45,7 @@ class TestDynamicFilterQueryFromString(): ]) def test_invalid_filter_strings(self, mock_get_host_model, filter_string): with pytest.raises(RuntimeError) as e: - DynamicFilter.query_from_string(filter_string) + SmartFilter.query_from_string(filter_string) assert e.value.message == u"Invalid query " + filter_string @pytest.mark.parametrize("filter_string,q_expected", [ @@ -53,7 +53,7 @@ class TestDynamicFilterQueryFromString(): (u'(ansible_facts__a=abc\u1F5E3def)', Q(**{u"ansible_facts__contains": {u"a": u"abc\u1F5E3def"}})), ]) def test_unicode(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) @pytest.mark.parametrize("filter_string,q_expected", [ @@ -67,7 +67,7 @@ class TestDynamicFilterQueryFromString(): ('a=b or a=d or a=e or a=z and b=h and b=i and b=j and b=k', Q(**{u"a": u"b"}) | Q(**{u"a": u"d"}) | Q(**{u"a": u"e"}) | Q(**{u"a": u"z"}) & Q(**{u"b": u"h"}) & Q(**{u"b": u"i"}) & Q(**{u"b": u"j"}) & Q(**{u"b": u"k"})) ]) def test_boolean_parenthesis(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) @pytest.mark.parametrize("filter_string,q_expected", [ @@ -87,7 +87,7 @@ class TestDynamicFilterQueryFromString(): #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})), ]) def test_contains_query_generated(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) @pytest.mark.parametrize("filter_string,q_expected", [ @@ -97,7 +97,7 @@ class TestDynamicFilterQueryFromString(): #('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})), ]) def test_contains_query_generated_unicode(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) @pytest.mark.parametrize("filter_string,q_expected", [ @@ -105,7 +105,7 @@ class TestDynamicFilterQueryFromString(): ('ansible_facts__c="null"', Q(**{u"ansible_facts__contains": {u"c": u"\"null\""}})), ]) def test_contains_query_generated_null(self, mock_get_host_model, filter_string, q_expected): - q = DynamicFilter.query_from_string(filter_string) + q = SmartFilter.query_from_string(filter_string) assert unicode(q) == unicode(q_expected) diff --git a/awx/main/utils/filters.py b/awx/main/utils/filters.py index 03ede25ea6..845d75a852 100644 --- a/awx/main/utils/filters.py +++ b/awx/main/utils/filters.py @@ -10,7 +10,7 @@ from pyparsing import ( import django -__all__ = ['DynamicFilter'] +__all__ = ['SmartFilter'] unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()] unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"'] @@ -35,7 +35,7 @@ def get_host_model(): return django.apps.apps.get_model('main', 'host') -class DynamicFilter(object): +class SmartFilter(object): SEARCHABLE_RELATIONSHIP = 'ansible_facts' class BoolOperand(object): @@ -68,20 +68,20 @@ class DynamicFilter(object): relationship refered to to see if it's a jsonb type. ''' def _json_path_to_contains(self, k, v): - if not k.startswith(DynamicFilter.SEARCHABLE_RELATIONSHIP): + if not k.startswith(SmartFilter.SEARCHABLE_RELATIONSHIP): v = self.strip_quotes_traditional_logic(v) return (k, v) # Strip off leading relationship key - if k.startswith(DynamicFilter.SEARCHABLE_RELATIONSHIP + '__'): - strip_len = len(DynamicFilter.SEARCHABLE_RELATIONSHIP) + 2 + if k.startswith(SmartFilter.SEARCHABLE_RELATIONSHIP + '__'): + strip_len = len(SmartFilter.SEARCHABLE_RELATIONSHIP) + 2 else: - strip_len = len(DynamicFilter.SEARCHABLE_RELATIONSHIP) + strip_len = len(SmartFilter.SEARCHABLE_RELATIONSHIP) k = k[strip_len:] pieces = k.split(u'__') - assembled_k = u'%s__contains' % (DynamicFilter.SEARCHABLE_RELATIONSHIP) + assembled_k = u'%s__contains' % (SmartFilter.SEARCHABLE_RELATIONSHIP) assembled_v = None last_v = None