mirror of
https://github.com/ansible/awx.git
synced 2026-01-11 01:57:35 -03:30
Use filtering/sorting from django-ansible-base (#14726)
* Move filtering to DAB * add comment to trigger building a new image Signed-off-by: jessicamack <jmack@redhat.com> * remove unneeded comment Signed-off-by: jessicamack <jmack@redhat.com> * remove unused imports Signed-off-by: jessicamack <jmack@redhat.com> * change mock import Signed-off-by: jessicamack <jmack@redhat.com> --------- Signed-off-by: jessicamack <jmack@redhat.com> Co-authored-by: jessicamack <jmack@redhat.com>
This commit is contained in:
John Westcott IV
GitHub
parent
325f5250db
commit
aacf9653c5
@ -1,450 +0,0 @@
|
||||
# Copyright (c) 2015 Ansible, Inc.
|
||||
# All Rights Reserved.
|
||||
|
||||
# Python
|
||||
import re
|
||||
import json
|
||||
from functools import reduce
|
||||
|
||||
# Django
|
||||
from django.core.exceptions import FieldError, ValidationError, FieldDoesNotExist
|
||||
from django.db import models
|
||||
from django.db.models import Q, CharField, IntegerField, BooleanField, TextField, JSONField
|
||||
from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
|
||||
from django.db.models.functions import Cast
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from django.contrib.contenttypes.fields import GenericForeignKey
|
||||
from django.utils.encoding import force_str
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
# Django REST Framework
|
||||
from rest_framework.exceptions import ParseError, PermissionDenied
|
||||
from rest_framework.filters import BaseFilterBackend
|
||||
|
||||
# AWX
|
||||
from awx.main.utils import get_type_for_model, to_python_boolean
|
||||
from awx.main.utils.db import get_all_field_names
|
||||
|
||||
|
||||
class TypeFilterBackend(BaseFilterBackend):
|
||||
"""
|
||||
Filter on type field now returned with all objects.
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
try:
|
||||
types = None
|
||||
for key, value in request.query_params.items():
|
||||
if key == 'type':
|
||||
if ',' in value:
|
||||
types = value.split(',')
|
||||
else:
|
||||
types = (value,)
|
||||
if types:
|
||||
types_map = {}
|
||||
for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
|
||||
ct_model = ct.model_class()
|
||||
if not ct_model:
|
||||
continue
|
||||
ct_type = get_type_for_model(ct_model)
|
||||
types_map[ct_type] = ct.pk
|
||||
model = queryset.model
|
||||
model_type = get_type_for_model(model)
|
||||
if 'polymorphic_ctype' in get_all_field_names(model):
|
||||
types_pks = set([v for k, v in types_map.items() if k in types])
|
||||
queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
|
||||
elif model_type in types:
|
||||
queryset = queryset
|
||||
else:
|
||||
queryset = queryset.none()
|
||||
return queryset
|
||||
except FieldError as e:
|
||||
# Return a 400 for invalid field names.
|
||||
raise ParseError(*e.args)
|
||||
|
||||
|
||||
def get_fields_from_path(model, path):
|
||||
"""
|
||||
Given a Django ORM lookup path (possibly over multiple models)
|
||||
Returns the fields in the line, and also the revised lookup path
|
||||
ex., given
|
||||
model=Organization
|
||||
path='project__timeout'
|
||||
returns tuple of fields traversed as well and a corrected path,
|
||||
for special cases we do substitutions
|
||||
([<IntegerField for timeout>], 'project__timeout')
|
||||
"""
|
||||
# Store of all the fields used to detect repeats
|
||||
field_list = []
|
||||
new_parts = []
|
||||
for name in path.split('__'):
|
||||
if model is None:
|
||||
raise ParseError(_('No related model for field {}.').format(name))
|
||||
# HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
|
||||
if model._meta.object_name in ('Project', 'InventorySource'):
|
||||
name = {'current_update': 'current_job', 'last_update': 'last_job', 'last_update_failed': 'last_job_failed', 'last_updated': 'last_job_run'}.get(
|
||||
name, name
|
||||
)
|
||||
|
||||
if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
|
||||
name = 'polymorphic_ctype'
|
||||
new_parts.append('polymorphic_ctype__model')
|
||||
else:
|
||||
new_parts.append(name)
|
||||
|
||||
if name in getattr(model, 'PASSWORD_FIELDS', ()):
|
||||
raise PermissionDenied(_('Filtering on password fields is not allowed.'))
|
||||
elif name == 'pk':
|
||||
field = model._meta.pk
|
||||
else:
|
||||
name_alt = name.replace("_", "")
|
||||
if name_alt in model._meta.fields_map.keys():
|
||||
field = model._meta.fields_map[name_alt]
|
||||
new_parts.pop()
|
||||
new_parts.append(name_alt)
|
||||
else:
|
||||
field = model._meta.get_field(name)
|
||||
if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
|
||||
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
|
||||
elif getattr(field, '__prevent_search__', False):
|
||||
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
|
||||
if field in field_list:
|
||||
# Field traversed twice, could create infinite JOINs, DoSing Tower
|
||||
raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
|
||||
field_list.append(field)
|
||||
model = getattr(field, 'related_model', None)
|
||||
|
||||
return field_list, '__'.join(new_parts)
|
||||
|
||||
|
||||
def get_field_from_path(model, path):
|
||||
"""
|
||||
Given a Django ORM lookup path (possibly over multiple models)
|
||||
Returns the last field in the line, and the revised lookup path
|
||||
ex.
|
||||
(<IntegerField for timeout>, 'project__timeout')
|
||||
"""
|
||||
field_list, new_path = get_fields_from_path(model, path)
|
||||
return (field_list[-1], new_path)
|
||||
|
||||
|
||||
class FieldLookupBackend(BaseFilterBackend):
|
||||
"""
|
||||
Filter using field lookups provided via query string parameters.
|
||||
"""
|
||||
|
||||
RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate', 'limit')
|
||||
|
||||
SUPPORTED_LOOKUPS = (
|
||||
'exact',
|
||||
'iexact',
|
||||
'contains',
|
||||
'icontains',
|
||||
'startswith',
|
||||
'istartswith',
|
||||
'endswith',
|
||||
'iendswith',
|
||||
'regex',
|
||||
'iregex',
|
||||
'gt',
|
||||
'gte',
|
||||
'lt',
|
||||
'lte',
|
||||
'in',
|
||||
'isnull',
|
||||
'search',
|
||||
)
|
||||
|
||||
# A list of fields that we know can be filtered on without the possibility
|
||||
# of introducing duplicates
|
||||
NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField)
|
||||
|
||||
def get_fields_from_lookup(self, model, lookup):
|
||||
if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS:
|
||||
path, suffix = lookup.rsplit('__', 1)
|
||||
else:
|
||||
path = lookup
|
||||
suffix = 'exact'
|
||||
|
||||
if not path:
|
||||
raise ParseError(_('Query string field name not provided.'))
|
||||
|
||||
# FIXME: Could build up a list of models used across relationships, use
|
||||
# those lookups combined with request.user.get_queryset(Model) to make
|
||||
# sure user cannot query using objects he could not view.
|
||||
field_list, new_path = get_fields_from_path(model, path)
|
||||
|
||||
new_lookup = new_path
|
||||
new_lookup = '__'.join([new_path, suffix])
|
||||
return field_list, new_lookup
|
||||
|
||||
def get_field_from_lookup(self, model, lookup):
|
||||
'''Method to match return type of single field, if needed.'''
|
||||
field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
|
||||
return (field_list[-1], new_lookup)
|
||||
|
||||
def to_python_related(self, value):
|
||||
value = force_str(value)
|
||||
if value.lower() in ('none', 'null'):
|
||||
return None
|
||||
else:
|
||||
return int(value)
|
||||
|
||||
def value_to_python_for_field(self, field, value):
|
||||
if isinstance(field, models.BooleanField):
|
||||
return to_python_boolean(value)
|
||||
elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
|
||||
try:
|
||||
return self.to_python_related(value)
|
||||
except ValueError:
|
||||
raise ParseError(_('Invalid {field_name} id: {field_id}').format(field_name=getattr(field, 'name', 'related field'), field_id=value))
|
||||
else:
|
||||
return field.to_python(value)
|
||||
|
||||
def value_to_python(self, model, lookup, value):
|
||||
try:
|
||||
lookup.encode("ascii")
|
||||
except UnicodeEncodeError:
|
||||
raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
|
||||
|
||||
field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
|
||||
field = field_list[-1]
|
||||
|
||||
needs_distinct = not all(isinstance(f, self.NO_DUPLICATES_ALLOW_LIST) for f in field_list)
|
||||
|
||||
# Type names are stored without underscores internally, but are presented and
|
||||
# and serialized over the API containing underscores so we remove `_`
|
||||
# for polymorphic_ctype__model lookups.
|
||||
if new_lookup.startswith('polymorphic_ctype__model'):
|
||||
value = value.replace('_', '')
|
||||
elif new_lookup.endswith('__isnull'):
|
||||
value = to_python_boolean(value)
|
||||
elif new_lookup.endswith('__in'):
|
||||
items = []
|
||||
if not value:
|
||||
raise ValueError('cannot provide empty value for __in')
|
||||
for item in value.split(','):
|
||||
items.append(self.value_to_python_for_field(field, item))
|
||||
value = items
|
||||
elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
|
||||
try:
|
||||
re.compile(value)
|
||||
except re.error as e:
|
||||
raise ValueError(e.args[0])
|
||||
elif new_lookup.endswith('__iexact'):
|
||||
if not isinstance(field, (CharField, TextField)):
|
||||
raise ValueError(f'{field.name} is not a text field and cannot be filtered by case-insensitive search')
|
||||
elif new_lookup.endswith('__search'):
|
||||
related_model = getattr(field, 'related_model', None)
|
||||
if not related_model:
|
||||
raise ValueError('%s is not searchable' % new_lookup[:-8])
|
||||
new_lookups = []
|
||||
for rm_field in related_model._meta.fields:
|
||||
if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
|
||||
new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
|
||||
return value, new_lookups, needs_distinct
|
||||
else:
|
||||
if isinstance(field, JSONField):
|
||||
new_lookup = new_lookup.replace(field.name, f'{field.name}_as_txt')
|
||||
value = self.value_to_python_for_field(field, value)
|
||||
return value, new_lookup, needs_distinct
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
try:
|
||||
# Apply filters specified via query_params. Each entry in the lists
|
||||
# below is (negate, field, value).
|
||||
and_filters = []
|
||||
or_filters = []
|
||||
chain_filters = []
|
||||
role_filters = []
|
||||
search_filters = {}
|
||||
needs_distinct = False
|
||||
# Can only have two values: 'AND', 'OR'
|
||||
# If 'AND' is used, an item must satisfy all conditions to show up in the results.
|
||||
# If 'OR' is used, an item just needs to satisfy one condition to appear in results.
|
||||
search_filter_relation = 'OR'
|
||||
for key, values in request.query_params.lists():
|
||||
if key in self.RESERVED_NAMES:
|
||||
continue
|
||||
|
||||
# HACK: make `created` available via API for the Django User ORM model
|
||||
# so it keep compatibility with other objects which exposes the `created` attr.
|
||||
if queryset.model._meta.object_name == 'User' and key.startswith('created'):
|
||||
key = key.replace('created', 'date_joined')
|
||||
|
||||
# HACK: Make job event filtering by host name mostly work even
|
||||
# when not capturing job event hosts M2M.
|
||||
if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
|
||||
key = key.replace('hosts__name', 'or__host__name')
|
||||
or_filters.append((False, 'host__name__isnull', True))
|
||||
|
||||
# Custom __int filter suffix (internal use only).
|
||||
q_int = False
|
||||
if key.endswith('__int'):
|
||||
key = key[:-5]
|
||||
q_int = True
|
||||
|
||||
# RBAC filtering
|
||||
if key == 'role_level':
|
||||
role_filters.append(values[0])
|
||||
continue
|
||||
|
||||
# Search across related objects.
|
||||
if key.endswith('__search'):
|
||||
if values and ',' in values[0]:
|
||||
search_filter_relation = 'AND'
|
||||
values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values])
|
||||
for value in values:
|
||||
search_value, new_keys, _ = self.value_to_python(queryset.model, key, force_str(value))
|
||||
assert isinstance(new_keys, list)
|
||||
search_filters[search_value] = new_keys
|
||||
# by definition, search *only* joins across relations,
|
||||
# so it _always_ needs a .distinct()
|
||||
needs_distinct = True
|
||||
continue
|
||||
|
||||
# Custom chain__ and or__ filters, mutually exclusive (both can
|
||||
# precede not__).
|
||||
q_chain = False
|
||||
q_or = False
|
||||
if key.startswith('chain__'):
|
||||
key = key[7:]
|
||||
q_chain = True
|
||||
elif key.startswith('or__'):
|
||||
key = key[4:]
|
||||
q_or = True
|
||||
|
||||
# Custom not__ filter prefix.
|
||||
q_not = False
|
||||
if key.startswith('not__'):
|
||||
key = key[5:]
|
||||
q_not = True
|
||||
|
||||
# Convert value(s) to python and add to the appropriate list.
|
||||
for value in values:
|
||||
if q_int:
|
||||
value = int(value)
|
||||
value, new_key, distinct = self.value_to_python(queryset.model, key, value)
|
||||
if distinct:
|
||||
needs_distinct = True
|
||||
if '_as_txt' in new_key:
|
||||
fname = next(item for item in new_key.split('__') if item.endswith('_as_txt'))
|
||||
queryset = queryset.annotate(**{fname: Cast(fname[:-7], output_field=TextField())})
|
||||
if q_chain:
|
||||
chain_filters.append((q_not, new_key, value))
|
||||
elif q_or:
|
||||
or_filters.append((q_not, new_key, value))
|
||||
else:
|
||||
and_filters.append((q_not, new_key, value))
|
||||
|
||||
# Now build Q objects for database query filter.
|
||||
if and_filters or or_filters or chain_filters or role_filters or search_filters:
|
||||
args = []
|
||||
for n, k, v in and_filters:
|
||||
if n:
|
||||
args.append(~Q(**{k: v}))
|
||||
else:
|
||||
args.append(Q(**{k: v}))
|
||||
for role_name in role_filters:
|
||||
if not hasattr(queryset.model, 'accessible_pk_qs'):
|
||||
raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.'))
|
||||
args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name)))
|
||||
if or_filters:
|
||||
q = Q()
|
||||
for n, k, v in or_filters:
|
||||
if n:
|
||||
q |= ~Q(**{k: v})
|
||||
else:
|
||||
q |= Q(**{k: v})
|
||||
args.append(q)
|
||||
if search_filters and search_filter_relation == 'OR':
|
||||
q = Q()
|
||||
for term, constrains in search_filters.items():
|
||||
for constrain in constrains:
|
||||
q |= Q(**{constrain: term})
|
||||
args.append(q)
|
||||
elif search_filters and search_filter_relation == 'AND':
|
||||
for term, constrains in search_filters.items():
|
||||
q_chain = Q()
|
||||
for constrain in constrains:
|
||||
q_chain |= Q(**{constrain: term})
|
||||
queryset = queryset.filter(q_chain)
|
||||
for n, k, v in chain_filters:
|
||||
if n:
|
||||
q = ~Q(**{k: v})
|
||||
else:
|
||||
q = Q(**{k: v})
|
||||
queryset = queryset.filter(q)
|
||||
queryset = queryset.filter(*args)
|
||||
if needs_distinct:
|
||||
queryset = queryset.distinct()
|
||||
return queryset
|
||||
except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
|
||||
raise ParseError(e.args[0])
|
||||
except ValidationError as e:
|
||||
raise ParseError(json.dumps(e.messages, ensure_ascii=False))
|
||||
|
||||
|
||||
class OrderByBackend(BaseFilterBackend):
|
||||
"""
|
||||
Filter to apply ordering based on query string parameters.
|
||||
"""
|
||||
|
||||
def filter_queryset(self, request, queryset, view):
|
||||
try:
|
||||
order_by = None
|
||||
for key, value in request.query_params.items():
|
||||
if key in ('order', 'order_by'):
|
||||
order_by = value
|
||||
if ',' in value:
|
||||
order_by = value.split(',')
|
||||
else:
|
||||
order_by = (value,)
|
||||
default_order_by = self.get_default_ordering(view)
|
||||
# glue the order by and default order by together so that the default is the backup option
|
||||
order_by = list(order_by or []) + list(default_order_by or [])
|
||||
if order_by:
|
||||
order_by = self._validate_ordering_fields(queryset.model, order_by)
|
||||
# Special handling of the type field for ordering. In this
|
||||
# case, we're not sorting exactly on the type field, but
|
||||
# given the limited number of views with multiple types,
|
||||
# sorting on polymorphic_ctype.model is effectively the same.
|
||||
new_order_by = []
|
||||
if 'polymorphic_ctype' in get_all_field_names(queryset.model):
|
||||
for field in order_by:
|
||||
if field == 'type':
|
||||
new_order_by.append('polymorphic_ctype__model')
|
||||
elif field == '-type':
|
||||
new_order_by.append('-polymorphic_ctype__model')
|
||||
else:
|
||||
new_order_by.append(field)
|
||||
else:
|
||||
for field in order_by:
|
||||
if field not in ('type', '-type'):
|
||||
new_order_by.append(field)
|
||||
queryset = queryset.order_by(*new_order_by)
|
||||
return queryset
|
||||
except FieldError as e:
|
||||
# Return a 400 for invalid field names.
|
||||
raise ParseError(*e.args)
|
||||
|
||||
def get_default_ordering(self, view):
|
||||
ordering = getattr(view, 'ordering', None)
|
||||
if isinstance(ordering, str):
|
||||
return (ordering,)
|
||||
return ordering
|
||||
|
||||
def _validate_ordering_fields(self, model, order_by):
|
||||
for field_name in order_by:
|
||||
# strip off the negation prefix `-` if it exists
|
||||
prefix = ''
|
||||
path = field_name
|
||||
if field_name[0] == '-':
|
||||
prefix = field_name[0]
|
||||
path = field_name[1:]
|
||||
try:
|
||||
field, new_path = get_field_from_path(model, path)
|
||||
new_path = '{}{}'.format(prefix, new_path)
|
||||
except (FieldError, FieldDoesNotExist) as e:
|
||||
raise ParseError(e.args[0])
|
||||
yield new_path
|
||||
@ -30,12 +30,13 @@ from rest_framework.permissions import IsAuthenticated
|
||||
from rest_framework.renderers import StaticHTMLRenderer
|
||||
from rest_framework.negotiation import DefaultContentNegotiation
|
||||
|
||||
from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
|
||||
from ansible_base.utils.models import get_all_field_names
|
||||
|
||||
# AWX
|
||||
from awx.api.filters import FieldLookupBackend
|
||||
from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
|
||||
from awx.main.access import optimize_queryset
|
||||
from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
|
||||
from awx.main.utils.db import get_all_field_names
|
||||
from awx.main.utils.licensing import server_product_name
|
||||
from awx.main.views import ApiErrorView
|
||||
from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer
|
||||
|
||||
@ -43,6 +43,8 @@ from rest_framework.utils.serializer_helpers import ReturnList
|
||||
# Django-Polymorphic
|
||||
from polymorphic.models import PolymorphicModel
|
||||
|
||||
from ansible_base.utils.models import get_type_for_model
|
||||
|
||||
# AWX
|
||||
from awx.main.access import get_user_capabilities
|
||||
from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE
|
||||
@ -102,7 +104,6 @@ from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
|
||||
from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry
|
||||
from awx.main.fields import ImplicitRoleField
|
||||
from awx.main.utils import (
|
||||
get_type_for_model,
|
||||
get_model_for_type,
|
||||
camelcase_to_underscore,
|
||||
getattrd,
|
||||
|
||||
@ -7,8 +7,10 @@ import json
|
||||
# Django
|
||||
from django.db import models
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.main.models.base import CreatedModifiedModel, prevent_search
|
||||
from awx.main.models.base import CreatedModifiedModel
|
||||
from awx.main.utils import encrypt_field
|
||||
from awx.conf import settings_registry
|
||||
|
||||
|
||||
@ -20,11 +20,12 @@ from rest_framework.exceptions import ParseError, PermissionDenied
|
||||
# Django OAuth Toolkit
|
||||
from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
|
||||
|
||||
from ansible_base.utils.validation import to_python_boolean
|
||||
|
||||
# AWX
|
||||
from awx.main.utils import (
|
||||
get_object_or_400,
|
||||
get_pk_from_dict,
|
||||
to_python_boolean,
|
||||
get_licenser,
|
||||
)
|
||||
from awx.main.models import (
|
||||
|
||||
@ -6,8 +6,10 @@ from django.conf import settings # noqa
|
||||
from django.db import connection
|
||||
from django.db.models.signals import pre_delete # noqa
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
|
||||
from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
|
||||
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa
|
||||
from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa
|
||||
from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa
|
||||
|
||||
@ -12,9 +12,11 @@ from django.utils.text import Truncator
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
from django.core.exceptions import ValidationError
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
|
||||
from awx.main.models.base import AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
|
||||
from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent
|
||||
from awx.main.models.unified_jobs import UnifiedJob
|
||||
from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate
|
||||
|
||||
@ -15,7 +15,6 @@ from awx.main.utils import encrypt_field, parse_yaml_or_json
|
||||
from awx.main.constants import CLOUD_PROVIDERS
|
||||
|
||||
__all__ = [
|
||||
'prevent_search',
|
||||
'VarsDictProperty',
|
||||
'BaseModel',
|
||||
'CreatedModifiedModel',
|
||||
@ -384,23 +383,6 @@ class NotificationFieldsModel(BaseModel):
|
||||
notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started')
|
||||
|
||||
|
||||
def prevent_search(relation):
|
||||
"""
|
||||
Used to mark a model field or relation as "restricted from filtering"
|
||||
e.g.,
|
||||
|
||||
class AuthToken(BaseModel):
|
||||
user = prevent_search(models.ForeignKey(...))
|
||||
sensitive_data = prevent_search(models.CharField(...))
|
||||
|
||||
The flag set by this function is used by
|
||||
`awx.api.filters.FieldLookupBackend` to block fields and relations that
|
||||
should not be searchable/filterable via search query params
|
||||
"""
|
||||
setattr(relation, '__prevent_search__', True)
|
||||
return relation
|
||||
|
||||
|
||||
def accepts_json(relation):
|
||||
"""
|
||||
Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars
|
||||
|
||||
@ -17,6 +17,8 @@ from django.db.models import Sum, Q
|
||||
import redis
|
||||
from solo.models import SingletonModel
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx import __version__ as awx_application_version
|
||||
from awx.main.utils import is_testing
|
||||
@ -24,7 +26,7 @@ from awx.api.versioning import reverse
|
||||
from awx.main.fields import ImplicitRoleField
|
||||
from awx.main.managers import InstanceManager, UUID_DEFAULT
|
||||
from awx.main.constants import JOB_FOLDER_PREFIX
|
||||
from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search
|
||||
from awx.main.models.base import BaseModel, HasEditsMixin
|
||||
from awx.main.models.rbac import (
|
||||
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
|
||||
ROLE_SINGLETON_SYSTEM_AUDITOR,
|
||||
|
||||
@ -25,6 +25,8 @@ from django.db.models import Q
|
||||
# REST Framework
|
||||
from rest_framework.exceptions import ParseError
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.constants import CLOUD_PROVIDERS
|
||||
@ -35,7 +37,7 @@ from awx.main.fields import (
|
||||
OrderedManyToManyField,
|
||||
)
|
||||
from awx.main.managers import HostManager, HostMetricActiveManager
|
||||
from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json
|
||||
from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
|
||||
from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
|
||||
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
|
||||
from awx.main.models.mixins import (
|
||||
|
||||
@ -20,13 +20,14 @@ from django.core.exceptions import FieldDoesNotExist
|
||||
# REST Framework
|
||||
from rest_framework.exceptions import ParseError
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.constants import HOST_FACTS_FIELDS
|
||||
from awx.main.models.base import (
|
||||
BaseModel,
|
||||
CreatedModifiedModel,
|
||||
prevent_search,
|
||||
accepts_json,
|
||||
JOB_TYPE_CHOICES,
|
||||
NEW_JOB_TYPE_CHOICES,
|
||||
|
||||
@ -16,8 +16,9 @@ from django.db.models.query import QuerySet
|
||||
from django.utils.crypto import get_random_string
|
||||
from django.utils.translation import gettext_lazy as _
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.main.models.base import prevent_search
|
||||
from awx.main.models.rbac import Role, RoleAncestorEntry
|
||||
from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
|
||||
from awx.main.utils.execution_environments import get_default_execution_environment
|
||||
|
||||
@ -15,9 +15,11 @@ from django.utils.encoding import smart_str, force_str
|
||||
from jinja2 import sandbox, ChainableUndefined
|
||||
from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search
|
||||
from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
|
||||
from awx.main.utils import encrypt_field, decrypt_field, set_environ
|
||||
from awx.main.notifications.email_backend import CustomEmailBackend
|
||||
from awx.main.notifications.slack_backend import SlackBackend
|
||||
|
||||
@ -30,8 +30,10 @@ from rest_framework.exceptions import ParseError
|
||||
# Django-Polymorphic
|
||||
from polymorphic.models import PolymorphicModel
|
||||
|
||||
from ansible_base.utils.models import prevent_search, get_type_for_model
|
||||
|
||||
# AWX
|
||||
from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search
|
||||
from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel
|
||||
from awx.main.dispatch import get_task_queuename
|
||||
from awx.main.dispatch.control import Control as ControlDispatcher
|
||||
from awx.main.registrar import activity_stream_registrar
|
||||
@ -42,7 +44,6 @@ from awx.main.utils.common import (
|
||||
_inventory_updates,
|
||||
copy_model_by_class,
|
||||
copy_m2m_relationships,
|
||||
get_type_for_model,
|
||||
parse_yaml_or_json,
|
||||
getattr_dne,
|
||||
ScheduleDependencyManager,
|
||||
|
||||
@ -23,9 +23,11 @@ from crum import get_current_user
|
||||
from jinja2 import sandbox
|
||||
from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
|
||||
|
||||
from ansible_base.utils.models import prevent_search
|
||||
|
||||
# AWX
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob
|
||||
from awx.main.models import accepts_json, UnifiedJobTemplate, UnifiedJob
|
||||
from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
|
||||
from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
|
||||
from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR
|
||||
|
||||
@ -17,6 +17,8 @@ from django.utils.timezone import now as tz_now
|
||||
from django.conf import settings
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
|
||||
from ansible_base.utils.models import get_type_for_model
|
||||
|
||||
# AWX
|
||||
from awx.main.dispatch.reaper import reap_job
|
||||
from awx.main.models import (
|
||||
@ -34,7 +36,6 @@ from awx.main.models import (
|
||||
from awx.main.scheduler.dag_workflow import WorkflowDAG
|
||||
from awx.main.utils.pglock import advisory_lock
|
||||
from awx.main.utils import (
|
||||
get_type_for_model,
|
||||
ScheduleTaskManager,
|
||||
ScheduleWorkflowManager,
|
||||
)
|
||||
|
||||
@ -2,12 +2,12 @@ from unittest import mock
|
||||
import pytest
|
||||
import json
|
||||
|
||||
from ansible_base.utils.models import get_type_for_model
|
||||
|
||||
from awx.api.versioning import reverse
|
||||
from awx.main.models.jobs import JobTemplate, Job
|
||||
from awx.main.models.activity_stream import ActivityStream
|
||||
from awx.main.access import JobTemplateAccess
|
||||
from awx.main.utils.common import get_type_for_model
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
||||
@ -3,15 +3,13 @@
|
||||
import pytest
|
||||
|
||||
# Django
|
||||
from django.core.exceptions import FieldDoesNotExist
|
||||
from rest_framework.exceptions import PermissionDenied
|
||||
|
||||
from rest_framework.exceptions import PermissionDenied, ParseError
|
||||
from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
|
||||
|
||||
from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path
|
||||
from awx.main.models import (
|
||||
AdHocCommand,
|
||||
ActivityStream,
|
||||
Credential,
|
||||
Job,
|
||||
JobTemplate,
|
||||
SystemJob,
|
||||
@ -20,88 +18,11 @@ from awx.main.models import (
|
||||
WorkflowJob,
|
||||
WorkflowJobTemplate,
|
||||
WorkflowJobOptions,
|
||||
InventorySource,
|
||||
JobEvent,
|
||||
)
|
||||
from awx.main.models.oauth import OAuth2Application
|
||||
from awx.main.models.jobs import JobOptions
|
||||
|
||||
|
||||
def test_related():
|
||||
field_lookup = FieldLookupBackend()
|
||||
lookup = '__'.join(['inventory', 'organization', 'pk'])
|
||||
field, new_lookup = field_lookup.get_field_from_lookup(InventorySource, lookup)
|
||||
print(field)
|
||||
print(new_lookup)
|
||||
|
||||
|
||||
def test_invalid_filter_key():
|
||||
field_lookup = FieldLookupBackend()
|
||||
# FieldDoesNotExist is caught and converted to ParseError by filter_queryset
|
||||
with pytest.raises(FieldDoesNotExist) as excinfo:
|
||||
field_lookup.value_to_python(JobEvent, 'event_data.task_action', 'foo')
|
||||
assert 'has no field named' in str(excinfo)
|
||||
|
||||
|
||||
def test_invalid_field_hop():
|
||||
with pytest.raises(ParseError) as excinfo:
|
||||
get_field_from_path(Credential, 'organization__description__user')
|
||||
assert 'No related model for' in str(excinfo)
|
||||
|
||||
|
||||
def test_invalid_order_by_key():
|
||||
field_order_by = OrderByBackend()
|
||||
with pytest.raises(ParseError) as excinfo:
|
||||
[f for f in field_order_by._validate_ordering_fields(JobEvent, ('event_data.task_action',))]
|
||||
assert 'has no field named' in str(excinfo)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(u"empty_value", [u'', ''])
|
||||
def test_empty_in(empty_value):
|
||||
field_lookup = FieldLookupBackend()
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
field_lookup.value_to_python(JobTemplate, 'project__name__in', empty_value)
|
||||
assert 'empty value for __in' in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(u"valid_value", [u'foo', u'foo,'])
|
||||
def test_valid_in(valid_value):
|
||||
field_lookup = FieldLookupBackend()
|
||||
value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__in', valid_value)
|
||||
assert 'foo' in value
|
||||
|
||||
|
||||
def test_invalid_field():
|
||||
invalid_field = u"ヽヾ"
|
||||
field_lookup = FieldLookupBackend()
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
field_lookup.value_to_python(WorkflowJobTemplate, invalid_field, 'foo')
|
||||
assert 'is not an allowed field name. Must be ascii encodable.' in str(excinfo.value)
|
||||
|
||||
|
||||
def test_valid_iexact():
|
||||
field_lookup = FieldLookupBackend()
|
||||
value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__iexact', 'foo')
|
||||
assert 'foo' in value
|
||||
|
||||
|
||||
def test_invalid_iexact():
|
||||
field_lookup = FieldLookupBackend()
|
||||
with pytest.raises(ValueError) as excinfo:
|
||||
field_lookup.value_to_python(Job, 'id__iexact', '1')
|
||||
assert 'is not a text field and cannot be filtered by case-insensitive search' in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize('lookup_suffix', ['', 'contains', 'startswith', 'in'])
|
||||
@pytest.mark.parametrize('password_field', Credential.PASSWORD_FIELDS)
|
||||
def test_filter_on_password_field(password_field, lookup_suffix):
|
||||
field_lookup = FieldLookupBackend()
|
||||
lookup = '__'.join(filter(None, [password_field, lookup_suffix]))
|
||||
with pytest.raises(PermissionDenied) as excinfo:
|
||||
field, new_lookup = field_lookup.get_field_from_lookup(Credential, lookup)
|
||||
assert 'not allowed' in str(excinfo.value)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'model, query',
|
||||
[
|
||||
@ -128,10 +49,3 @@ def test_filter_sensitive_fields_and_relations(model, query):
|
||||
with pytest.raises(PermissionDenied) as excinfo:
|
||||
field, new_lookup = field_lookup.get_field_from_lookup(model, query)
|
||||
assert 'not allowed' in str(excinfo.value)
|
||||
|
||||
|
||||
def test_looping_filters_prohibited():
|
||||
field_lookup = FieldLookupBackend()
|
||||
with pytest.raises(ParseError) as loop_exc:
|
||||
field_lookup.get_field_from_lookup(Job, 'job_events__job__job_events')
|
||||
assert 'job_events' in str(loop_exc.value)
|
||||
|
||||
@ -12,6 +12,8 @@ from unittest import mock
|
||||
|
||||
from rest_framework.exceptions import ParseError
|
||||
|
||||
from ansible_base.utils.models import get_type_for_model
|
||||
|
||||
from awx.main.utils import common
|
||||
from awx.api.validators import HostnameRegexValidator
|
||||
|
||||
@ -106,7 +108,7 @@ TEST_MODELS = [
|
||||
# Cases relied on for scheduler dependent jobs list
|
||||
@pytest.mark.parametrize('model,name', TEST_MODELS)
|
||||
def test_get_type_for_model(model, name):
|
||||
assert common.get_type_for_model(model) == name
|
||||
assert get_type_for_model(model) == name
|
||||
|
||||
|
||||
def test_get_model_for_invalid_type():
|
||||
|
||||
@ -68,7 +68,9 @@ class mockHost:
|
||||
|
||||
@mock.patch('awx.main.utils.filters.get_model', return_value=mockHost())
|
||||
class TestSmartFilterQueryFromString:
|
||||
@mock.patch('awx.api.filters.get_fields_from_path', lambda model, path: ([model], path)) # disable field filtering, because a__b isn't a real Host field
|
||||
@mock.patch(
|
||||
'ansible_base.filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path: ([model], path)
|
||||
) # disable field filtering, because a__b isn't a real Host field
|
||||
@pytest.mark.parametrize(
|
||||
"filter_string,q_expected",
|
||||
[
|
||||
|
||||
@ -52,12 +52,10 @@ __all__ = [
|
||||
'get_awx_http_client_headers',
|
||||
'get_awx_version',
|
||||
'update_scm_url',
|
||||
'get_type_for_model',
|
||||
'get_model_for_type',
|
||||
'copy_model_by_class',
|
||||
'copy_m2m_relationships',
|
||||
'prefetch_page_capabilities',
|
||||
'to_python_boolean',
|
||||
'datetime_hook',
|
||||
'ignore_inventory_computed_fields',
|
||||
'ignore_inventory_group_removal',
|
||||
@ -110,18 +108,6 @@ def get_object_or_400(klass, *args, **kwargs):
|
||||
raise ParseError(*e.args)
|
||||
|
||||
|
||||
def to_python_boolean(value, allow_none=False):
|
||||
value = str(value)
|
||||
if value.lower() in ('true', '1', 't'):
|
||||
return True
|
||||
elif value.lower() in ('false', '0', 'f'):
|
||||
return False
|
||||
elif allow_none and value.lower() in ('none', 'null'):
|
||||
return None
|
||||
else:
|
||||
raise ValueError(_(u'Unable to convert "%s" to boolean') % value)
|
||||
|
||||
|
||||
def datetime_hook(d):
|
||||
new_d = {}
|
||||
for key, value in d.items():
|
||||
@ -569,14 +555,6 @@ def copy_m2m_relationships(obj1, obj2, fields, kwargs=None):
|
||||
dest_field.add(*list(src_field_value.all().values_list('id', flat=True)))
|
||||
|
||||
|
||||
def get_type_for_model(model):
|
||||
"""
|
||||
Return type name for a given model class.
|
||||
"""
|
||||
opts = model._meta.concrete_model._meta
|
||||
return camelcase_to_underscore(opts.object_name)
|
||||
|
||||
|
||||
def get_model_for_type(type_name):
|
||||
"""
|
||||
Return model class for a given type name.
|
||||
|
||||
@ -1,27 +1,10 @@
|
||||
# Copyright (c) 2017 Ansible by Red Hat
|
||||
# All Rights Reserved.
|
||||
|
||||
from itertools import chain
|
||||
|
||||
from awx.settings.application_name import set_application_name
|
||||
from django.conf import settings
|
||||
|
||||
|
||||
def get_all_field_names(model):
|
||||
# Implements compatibility with _meta.get_all_field_names
|
||||
# See: https://docs.djangoproject.com/en/1.11/ref/models/meta/#migrating-from-the-old-api
|
||||
return list(
|
||||
set(
|
||||
chain.from_iterable(
|
||||
(field.name, field.attname) if hasattr(field, 'attname') else (field.name,)
|
||||
for field in model._meta.get_fields()
|
||||
# For complete backwards compatibility, you may want to exclude
|
||||
# GenericForeignKey from the results.
|
||||
if not (field.many_to_one and field.related_model is None)
|
||||
)
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def set_connection_name(function):
|
||||
set_application_name(settings.DATABASES, settings.CLUSTER_HOST_ID, function=function)
|
||||
|
||||
@ -161,7 +161,7 @@ class SmartFilter(object):
|
||||
else:
|
||||
# detect loops and restrict access to sensitive fields
|
||||
# this import is intentional here to avoid a circular import
|
||||
from awx.api.filters import FieldLookupBackend
|
||||
from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
|
||||
|
||||
FieldLookupBackend().get_field_from_lookup(Host, k)
|
||||
kwargs[k] = v
|
||||
|
||||
@ -11,6 +11,7 @@ from datetime import timedelta
|
||||
|
||||
# python-ldap
|
||||
import ldap
|
||||
from split_settings.tools import include
|
||||
|
||||
|
||||
DEBUG = True
|
||||
@ -354,12 +355,6 @@ REST_FRAMEWORK = {
|
||||
'awx.api.authentication.LoggedBasicAuthentication',
|
||||
),
|
||||
'DEFAULT_PERMISSION_CLASSES': ('awx.api.permissions.ModelAccessPermission',),
|
||||
'DEFAULT_FILTER_BACKENDS': (
|
||||
'awx.api.filters.TypeFilterBackend',
|
||||
'awx.api.filters.FieldLookupBackend',
|
||||
'rest_framework.filters.SearchFilter',
|
||||
'awx.api.filters.OrderByBackend',
|
||||
),
|
||||
'DEFAULT_PARSER_CLASSES': ('awx.api.parsers.JSONParser',),
|
||||
'DEFAULT_RENDERER_CLASSES': ('awx.api.renderers.DefaultJSONRenderer', 'awx.api.renderers.BrowsableAPIRenderer'),
|
||||
'DEFAULT_METADATA_CLASS': 'awx.api.metadata.Metadata',
|
||||
@ -1067,3 +1062,12 @@ CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36 # months
|
||||
# Host metric summary monthly task - last time of run
|
||||
HOST_METRIC_SUMMARY_TASK_LAST_TS = None
|
||||
HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days
|
||||
|
||||
|
||||
# django-ansible-base
|
||||
ANSIBLE_BASE_FEATURES = {'AUTHENTICATION': False, 'SWAGGER': False, 'FILTERING': True}
|
||||
|
||||
from ansible_base import settings # noqa: E402
|
||||
|
||||
settings_file = os.path.join(os.path.dirname(settings.__file__), 'dynamic_settings.py')
|
||||
include(settings_file)
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user