External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-22 06:58:06 -03:30
Code Issues Packages Projects Releases Wiki Activity

Clean-up intiail commit for Host filter / DynamicInventory

Browse Source
This commit is contained in:
Wayne Witzel III 2017-04-24 15:45:12 -04:00
parent 932f53b2ea
commit 17e9b3057e
8 changed files with 309 additions and 214 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
awx/api/views.py
View File

@ -79,7 +79,7 @@ from awx.api.metadata import RoleMetadata
from awx.main.consumers import emit_channel_notification
from awx.main.models.unified_jobs import ACTIVE_STATES
from awx.main.scheduler.tasks import run_job_complete
from awx.main.fields import DynamicFilterField
from awx.main.querysets import DynamicFilterQuerySet
logger = logging.getLogger('awx.api.views')
@ -1764,10 +1764,10 @@ class HostList(ListCreateAPIView):
capabilities_prefetch = ['inventory.admin']
def get_queryset(self):
qs = super(HostList, self).get_queryset()
qs = DynamicFilterQuerySet(HostList, using=self._db)
filter_string = self.request.query_params.get('host_filter', None)
if filter_string:
filter_q = DynamicFilterField.filter_string_to_q(filter_string)
filter_q = qs.query_from_string(filter_string)
qs = qs.filter(filter_q)
return qs

206
awx/main/fields.py
View File

@ -4,10 +4,7 @@
# Python
import copy
import json
import re
import sys
import six
from pyparsing import infixNotation, opAssoc, Optional, Literal, CharsNotIn
from jinja2 import Environment, StrictUndefined
from jinja2.exceptions import UndefinedError
@ -28,7 +25,6 @@ from django.db.models.fields.related import (
ReverseManyRelatedObjectsDescriptor,
)
from django.utils.encoding import smart_text
from django.db.models import Q
from django.utils.translation import ugettext_lazy as _
# jsonschema
@ -39,6 +35,7 @@ from jsonfield import JSONField as upstream_JSONField
from jsonbfield.fields import JSONField as upstream_JSONBField
# AWX
from awx.main.querysets import DynamicFilterQuerySet
from awx.main.models.rbac import batch_role_ancestor_rebuilding, Role
from awx.main import utils
@ -332,202 +329,15 @@ class ImplicitRoleField(models.ForeignKey):
Role.rebuild_role_ancestor_list([], child_ids)
unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
def string_to_type(t):
if t == u'true':
return True
elif t == u'false':
return False
if re.search('^[-+]?[0-9]+$',t):
return int(t)
if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
return float(t)
return t
class DynamicFilterField(models.TextField):
SEARCHABLE_RELATIONSHIP = 'ansible_facts'
class BoolOperand(object):
def __init__(self, t):
kwargs = dict()
k, v = self._extract_key_value(t)
k, v = self._json_path_to_contains(k, v)
kwargs[k] = v
self.result = Q(**kwargs)
def strip_quotes_traditional_logic(self, v):
if type(v) is unicode and v.startswith('"') and v.endswith('"'):
return v[1:-1]
return v
def strip_quotes_json_logic(self, v):
if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
return v[1:-1]
return v
'''
TODO: We should be able to express this in the grammar and let
pyparsing do the heavy lifting.
TODO: separate django filter requests from our custom json filter
request so we don't process the key any. This could be
accomplished using a whitelist or introspecting the
relationship refered to to see if it's a jsonb type.
'''
def _json_path_to_contains(self, k, v):
if not k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP):
v = self.strip_quotes_traditional_logic(v)
return (k, v)
# Strip off leading relationship key
if k.startswith(DynamicFilterField.SEARCHABLE_RELATIONSHIP + '__'):
strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP) + 2
else:
strip_len = len(DynamicFilterField.SEARCHABLE_RELATIONSHIP)
k = k[strip_len:]
pieces = k.split(u'__')
assembled_k = u'%s__contains' % (DynamicFilterField.SEARCHABLE_RELATIONSHIP)
assembled_v = None
last_v = None
last_kv = None
for i, piece in enumerate(pieces):
new_kv = dict()
if piece.endswith(u'[]'):
new_v = []
new_kv[piece[0:-2]] = new_v
else:
new_v = dict()
new_kv[piece] = new_v
if last_kv is None:
assembled_v = new_kv
elif type(last_v) is list:
last_v.append(new_kv)
elif type(last_v) is dict:
last_kv[last_kv.keys()[0]] = new_kv
last_v = new_v
last_kv = new_kv
v = self.strip_quotes_json_logic(v)
if type(last_v) is list:
last_v.append(v)
elif type(last_v) is dict:
last_kv[last_kv.keys()[0]] = v
return (assembled_k, assembled_v)
def _extract_key_value(self, t):
t_len = len(t)
k = None
v = None
# key
# "something"=
v_offset = 2
if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
k = t[1]
v_offset = 4
# something=
else:
k = t[0]
# value
# ="something"
if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
v = u'"' + unicode(t[v_offset + 1]) + u'"'
#v = t[v_offset + 1]
# empty ""
elif t_len > (v_offset + 1):
v = u""
# no ""
else:
v = string_to_type(t[v_offset])
return (k, v)
class BoolBinOp(object):
def __init__(self, t):
self.result = None
i = 2
while i < len(t[0]):
if not self.result:
self.result = t[0][0].result
right = t[0][i].result
self.result = self.execute_logic(self.result, right)
i += 2
class BoolAnd(BoolBinOp):
def execute_logic(self, left, right):
return left & right
class BoolOr(BoolBinOp):
def execute_logic(self, left, right):
return left | right
class BoolNot(object):
def __init__(self, t):
self.right = t[0][1].result
self.result = self.execute_logic(self.right)
def execute_logic(self, right):
return ~right
@classmethod
def filter_string_to_q(cls, filter_string):
'''
TODO:
* handle values with " via: a.b.c.d="hello\"world"
* handle keys with " via: a.\"b.c="yeah"
* handle key with __ in it
'''
filter_string_raw = filter_string
filter_string = unicode(filter_string)
atom = CharsNotIn(unicode_spaces_other)
atom_inside_quotes = CharsNotIn(u'"')
atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
EQUAL = Literal('=')
grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
grammar.setParseAction(cls.BoolOperand)
boolExpr = infixNotation(grammar, [
("not", 1, opAssoc.RIGHT, cls.BoolNot),
("and", 2, opAssoc.LEFT, cls.BoolAnd),
("or", 2, opAssoc.LEFT, cls.BoolOr),
])
def get_prep_value(self, value):
if value is None:
return value
try:
res = boolExpr.parseString('(' + filter_string + ')')
#except ParseException as e:
except Exception:
raise RuntimeError(u"Invalid query %s" % filter_string_raw)
if len(res) > 0:
return res[0].result
raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)
DynamicFilterQuerySet().query_from_string(value)
except RuntimeError, e:
raise models.base.ValidationError(e)
return super(DynamicFilterField, self).get_prep_value(value)
class JSONSchemaField(JSONBField):

20
awx/main/managers.py
View File

@ -9,6 +9,8 @@ from django.utils.timezone import now
from django.db.models import Sum
from django.conf import settings
from awx.main.querysets import DynamicFilterQuerySet
class HostManager(models.Manager):
"""Custom manager class for Hosts model."""
@ -20,6 +22,24 @@ class HostManager(models.Manager):
except NotImplementedError: # For unit tests only, SQLite doesn't support distinct('name')
return len(set(self.values_list('name', flat=True)))
def get_queryset(self):
"""When the Inventory this host belongs to has a `host_filter` set
generate the QuerySet using that filter. Otherwise just return the default filter.
"""
qs = DynamicFilterQuerySet(self.model, using=self._db)
if self.instance is not None:
if hasattr(self.instance, 'host_filter') and self.instance.host_filter is not None:
q = qs.query_from_string(self.instance.host_filter)
# If we are using host_filters, disable the core_filters, this allows
# us to access all of the available Host entries, not just the ones associated
# with a specific FK/relation.
#
# If we don't disable this, a filter of {'inventory': self.instance} gets automatically
# injected by the related object mapper.
self.core_filters = {}
return qs.filter(q)
return qs
class InstanceManager(models.Manager):
"""A custom manager class for the Instance model.

5
awx/main/migrations/0038_v320_release.py
View File

@ -36,6 +36,11 @@ class Migration(migrations.Migration):
name='inventory',
field=models.ForeignKey(related_name='inventory_sources', default=None, to='main.Inventory', null=True),
),
migrations.AddField(
model_name='inventory',
name='host_filter',
field=awx.main.fields.DynamicFilterField(default=None, help_text='Filter that will be applied to the hosts of this inventory.', null=True, blank=True),
),
# Facts
migrations.AlterField(

13
awx/main/models/inventory.py
View File

@ -20,7 +20,11 @@ from django.utils.timezone import now
# AWX
from awx.api.versioning import reverse
from awx.main.constants import CLOUD_PROVIDERS
from awx.main.fields import ImplicitRoleField, JSONBField
from awx.main.fields import (
ImplicitRoleField,
JSONBField,
DynamicFilterField,
)
from awx.main.managers import HostManager
from awx.main.models.base import * # noqa
from awx.main.models.unified_jobs import * # noqa
@ -99,6 +103,13 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin):
editable=False,
help_text=_('Number of external inventory sources in this inventory with failures.'),
)
host_filter = DynamicFilterField(
blank=True,
null=True,
default=None,
help_text=_('Filter that will be applied to the hosts of this inventory.'),
)
admin_role = ImplicitRoleField(
parent_role='organization.admin_role',
)

218
awx/main/querysets.py Normal file
View File

@ -0,0 +1,218 @@
import re
import sys
from pyparsing import (
infixNotation,
opAssoc,
Optional,
Literal,
CharsNotIn,
)
from django.db import models
__all__ = ['DynamicFilterQuerySet']
unicode_spaces = [unichr(c) for c in xrange(sys.maxunicode) if unichr(c).isspace()]
unicode_spaces_other = unicode_spaces + [u'(', u')', u'=', u'"']
def string_to_type(t):
if t == u'true':
return True
elif t == u'false':
return False
if re.search('^[-+]?[0-9]+$',t):
return int(t)
if re.search('^[-+]?[0-9]+\.[0-9]+$',t):
return float(t)
return t
class DynamicFilterQuerySet(models.QuerySet):
class BoolOperand(object):
def __init__(self, t):
kwargs = dict()
k, v = self._extract_key_value(t)
k, v = self._json_path_to_contains(k, v)
kwargs[k] = v
self.result = models.Q(**kwargs)
'''
TODO: We should be able to express this in the grammar and let
pyparsing do the heavy lifting.
TODO: separate django filter requests from our custom json filter
request so we don't process the key any. This could be
accomplished using a whitelist or introspecting the
relationship refered to to see if it's a jsonb type.
'''
def _json_path_to_contains(self, k, v):
pieces = k.split('__')
flag_first_arr_found = False
assembled_k = ''
assembled_v = v
last_kv = None
last_v = None
contains_count = 0
for i, piece in enumerate(pieces):
if flag_first_arr_found is False and piece.endswith('[]'):
assembled_k += u'%s__contains' % (piece[0:-2])
contains_count += 1
flag_first_arr_found = True
elif flag_first_arr_found is False and i == len(pieces) - 1:
assembled_k += u'%s' % piece
elif flag_first_arr_found is False:
assembled_k += u'%s__' % piece
elif flag_first_arr_found is True:
new_kv = dict()
if piece.endswith('[]'):
new_v = []
new_kv[piece[0:-2]] = new_v
else:
new_v = dict()
new_kv[piece] = new_v
if last_v is None:
last_v = []
assembled_v = last_v
if type(last_v) is list:
last_v.append(new_kv)
elif type(last_v) is dict:
last_kv[last_kv.keys()[0]] = new_kv
last_v = new_v
last_kv = new_kv
contains_count += 1
'''
Explicit quotes are kept until this point.
Note: we could have totally "ripped" them off earlier when we decided
what type to convert the token to.
'''
if type(v) is unicode and v.startswith('"') and v.endswith('"') and v != u'"null"':
v = v[1:-1]
if contains_count == 0:
assembled_v = v
elif contains_count == 1:
assembled_v = [v]
elif contains_count > 1:
if type(last_v) is list:
last_v.append(v)
if type(last_v) is dict:
last_kv[last_kv.keys()[0]] = v
return (assembled_k, assembled_v)
def _extract_key_value(self, t):
t_len = len(t)
k = None
v = None
# key
# "something"=
v_offset = 2
if t_len >= 2 and t[0] == "\"" and t[2] == "\"":
k = t[1]
v_offset = 4
# something=
else:
k = t[0]
# value
# ="something"
if t_len > (v_offset + 2) and t[v_offset] == "\"" and t[v_offset + 2] == "\"":
v = u'"' + unicode(t[v_offset + 1]) + u'"'
#v = t[v_offset + 1]
# empty ""
elif t_len > (v_offset + 1):
v = u""
# no ""
else:
v = string_to_type(t[v_offset])
return (k, v)
class BoolBinOp(object):
def __init__(self, t):
self.result = None
i = 2
while i < len(t[0]):
if not self.result:
self.result = t[0][0].result
right = t[0][i].result
self.result = self.execute_logic(self.result, right)
i += 2
class BoolAnd(BoolBinOp):
def execute_logic(self, left, right):
return left & right
class BoolOr(BoolBinOp):
def execute_logic(self, left, right):
return left | right
class BoolNot(object):
def __init__(self, t):
self.right = t[0][1].result
self.result = self.execute_logic(self.right)
def execute_logic(self, right):
return ~right
@classmethod
def query_from_string(cls, filter_string):
'''
TODO:
* handle values with " via: a.b.c.d="hello\"world"
* handle keys with " via: a.\"b.c="yeah"
* handle key with __ in it
'''
filter_string_raw = filter_string
filter_string = unicode(filter_string)
atom = CharsNotIn(unicode_spaces_other)
atom_inside_quotes = CharsNotIn(u'"')
atom_quoted = Literal('"') + Optional(atom_inside_quotes) + Literal('"')
EQUAL = Literal('=')
grammar = ((atom_quoted | atom) + EQUAL + Optional((atom_quoted | atom)))
grammar.setParseAction(cls.BoolOperand)
boolExpr = infixNotation(grammar, [
("not", 1, opAssoc.RIGHT, cls.BoolNot),
("and", 2, opAssoc.LEFT, cls.BoolAnd),
("or", 2, opAssoc.LEFT, cls.BoolOr),
])
try:
res = boolExpr.parseString('(' + filter_string + ')')
except Exception:
raise RuntimeError(u"Invalid query %s" % filter_string_raw)
if len(res) > 0:
return res[0].result
raise RuntimeError("Parsing the filter_string %s went terribly wrong" % filter_string)

34
awx/main/tests/functional/models/test_inventory.py
View File

@ -2,7 +2,11 @@ import pytest
import mock
# AWX
from awx.main.models import InventorySource, InventoryUpdate
from awx.main.models import (
Inventory,
InventorySource,
InventoryUpdate,
)
@pytest.mark.django_db
@ -31,3 +35,31 @@ class TestSCMUpdateFeatures:
scm_inventory_source.description = "I'm testing this!"
scm_inventory_source.save()
assert not mck_update.called
@pytest.mark.django_db
def test_host_objects_manager(organization):
dynamic_inventory = Inventory(organization=organization, name='dynamic', host_filter='inventory_sources__source=ec2')
dynamic_inventory.save()
ec2_inv = Inventory(name='test_ec2', organization=organization)
ec2_inv.save()
ec2_source = ec2_inv.inventory_sources.create(name='test_ec2_source', source='ec2')
for i in range(2):
ec2_host = ec2_inv.hosts.create(name='test_ec2_{0}'.format(i))
ec2_host.inventory_sources.add(ec2_source)
ec2_inv.save()
gce_inv = Inventory(name='test_gce', organization=organization)
gce_inv.save()
gce_source = gce_inv.inventory_sources.create(name='test_gce_source', source='gce')
gce_host = gce_inv.hosts.create(name='test_gce_host')
gce_host.inventory_sources.add(gce_source)
gce_inv.save()
hosts = dynamic_inventory.hosts.all()
assert len(hosts) == 2
assert hosts[0].inventory_sources.first() == ec2_source
assert hosts[1].inventory_sources.first() == ec2_source

21
awx/main/tests/unit/test_fields.py → awx/main/tests/unit/test_querysets.py
View File

@ -3,14 +3,13 @@
import pytest
# AWX
from awx.main.fields import DynamicFilterField
from awx.main.querysets import DynamicFilterQuerySet
# Django
from django.db.models import Q
class TestDynamicFilterFieldFilterStringToQ():
class TestDynamicFilterQuerySetQueryFromString():
@pytest.mark.parametrize("filter_string,q_expected", [
('facts__facts__blank=""', Q(**{u"facts__facts__blank": u""})),
('"facts__facts__ space "="f"', Q(**{u"facts__facts__ space ": u"f"})),
@ -24,7 +23,7 @@ class TestDynamicFilterFieldFilterStringToQ():
#('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
])
def test_query_generated(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
@pytest.mark.parametrize("filter_string", [
@ -33,7 +32,7 @@ class TestDynamicFilterFieldFilterStringToQ():
])
def test_invalid_filter_strings(self, filter_string):
with pytest.raises(RuntimeError) as e:
DynamicFilterField.filter_string_to_q(filter_string)
DynamicFilterQuerySet.query_from_string(filter_string)
assert e.value.message == u"Invalid query " + filter_string
@pytest.mark.parametrize("filter_string,q_expected", [
@ -41,7 +40,7 @@ class TestDynamicFilterFieldFilterStringToQ():
(u'(ansible_facts__a=abc\u1F5E3def)', Q(**{u"ansible_facts__contains": {u"a": u"abc\u1F5E3def"}})),
])
def test_unicode(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
@pytest.mark.parametrize("filter_string,q_expected", [
@ -55,7 +54,7 @@ class TestDynamicFilterFieldFilterStringToQ():
('a=b or a=d or a=e or a=z and b=h and b=i and b=j and b=k', Q(**{u"a": u"b"}) | Q(**{u"a": u"d"}) | Q(**{u"a": u"e"}) | Q(**{u"a": u"z"}) & Q(**{u"b": u"h"}) & Q(**{u"b": u"i"}) & Q(**{u"b": u"j"}) & Q(**{u"b": u"k"}))
])
def test_boolean_parenthesis(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
@pytest.mark.parametrize("filter_string,q_expected", [
@ -75,7 +74,7 @@ class TestDynamicFilterFieldFilterStringToQ():
#('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
])
def test_contains_query_generated(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
@pytest.mark.parametrize("filter_string,q_expected", [
@ -85,7 +84,7 @@ class TestDynamicFilterFieldFilterStringToQ():
#('a__b\"__c="true"', Q(**{u"a__b\"__c": "true"})),
])
def test_contains_query_generated_unicode(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
@pytest.mark.parametrize("filter_string,q_expected", [
@ -93,12 +92,12 @@ class TestDynamicFilterFieldFilterStringToQ():
('ansible_facts__c="null"', Q(**{u"ansible_facts__contains": {u"c": u"\"null\""}})),
])
def test_contains_query_generated_null(self, filter_string, q_expected):
q = DynamicFilterField.filter_string_to_q(filter_string)
q = DynamicFilterQuerySet.query_from_string(filter_string)
assert unicode(q) == unicode(q_expected)
'''
#('"facts__quoted_val"="f\"oo"', 1),
#('facts__facts__arr[]="foo"', 1),
#('facts__facts__arr_nested[]__a[]="foo"', 1),
#('facts__facts__arr_nested[]__a[]="foo"', 1),
'''