External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-05-07 09:27:36 -02:30
Code Issues Packages Projects Releases Wiki Activity

Use filtering/sorting from django-ansible-base (#14726)

Browse Source 
* Move filtering to DAB

* add comment to trigger building a new image

Signed-off-by: jessicamack <jmack@redhat.com>

* remove unneeded comment

Signed-off-by: jessicamack <jmack@redhat.com>

* remove unused imports

Signed-off-by: jessicamack <jmack@redhat.com>

* change mock import

Signed-off-by: jessicamack <jmack@redhat.com>

---------

Signed-off-by: jessicamack <jmack@redhat.com>
Co-authored-by: jessicamack <jmack@redhat.com>
This commit is contained in:
John Westcott IV
2023-12-18 10:05:02 -05:00
committed by GitHub
parent 325f5250db
commit aacf9653c5
24 changed files with 57 additions and 621 deletions
Download Patch File Download Diff File Expand all files Collapse all files

450
awx/api/filters.py
View File

@@ -1,450 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
# Python
import re
import json
from functools import reduce
# Django
from django.core.exceptions import FieldError, ValidationError, FieldDoesNotExist
from django.db import models
from django.db.models import Q, CharField, IntegerField, BooleanField, TextField, JSONField
from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey
from django.db.models.functions import Cast
from django.contrib.contenttypes.models import ContentType
from django.contrib.contenttypes.fields import GenericForeignKey
from django.utils.encoding import force_str
from django.utils.translation import gettext_lazy as _
# Django REST Framework
from rest_framework.exceptions import ParseError, PermissionDenied
from rest_framework.filters import BaseFilterBackend
# AWX
from awx.main.utils import get_type_for_model, to_python_boolean
from awx.main.utils.db import get_all_field_names
class TypeFilterBackend(BaseFilterBackend):
"""
Filter on type field now returned with all objects.
"""
def filter_queryset(self, request, queryset, view):
try:
types = None
for key, value in request.query_params.items():
if key == 'type':
if ',' in value:
types = value.split(',')
else:
types = (value,)
if types:
types_map = {}
for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')):
ct_model = ct.model_class()
if not ct_model:
continue
ct_type = get_type_for_model(ct_model)
types_map[ct_type] = ct.pk
model = queryset.model
model_type = get_type_for_model(model)
if 'polymorphic_ctype' in get_all_field_names(model):
types_pks = set([v for k, v in types_map.items() if k in types])
queryset = queryset.filter(polymorphic_ctype_id__in=types_pks)
elif model_type in types:
queryset = queryset
else:
queryset = queryset.none()
return queryset
except FieldError as e:
# Return a 400 for invalid field names.
raise ParseError(*e.args)
def get_fields_from_path(model, path):
"""
Given a Django ORM lookup path (possibly over multiple models)
Returns the fields in the line, and also the revised lookup path
ex., given
model=Organization
path='project__timeout'
returns tuple of fields traversed as well and a corrected path,
for special cases we do substitutions
([<IntegerField for timeout>], 'project__timeout')
"""
# Store of all the fields used to detect repeats
field_list = []
new_parts = []
for name in path.split('__'):
if model is None:
raise ParseError(_('No related model for field {}.').format(name))
# HACK: Make project and inventory source filtering by old field names work for backwards compatibility.
if model._meta.object_name in ('Project', 'InventorySource'):
name = {'current_update': 'current_job', 'last_update': 'last_job', 'last_update_failed': 'last_job_failed', 'last_updated': 'last_job_run'}.get(
name, name
)
if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model):
name = 'polymorphic_ctype'
new_parts.append('polymorphic_ctype__model')
else:
new_parts.append(name)
if name in getattr(model, 'PASSWORD_FIELDS', ()):
raise PermissionDenied(_('Filtering on password fields is not allowed.'))
elif name == 'pk':
field = model._meta.pk
else:
name_alt = name.replace("_", "")
if name_alt in model._meta.fields_map.keys():
field = model._meta.fields_map[name_alt]
new_parts.pop()
new_parts.append(name_alt)
else:
field = model._meta.get_field(name)
if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False):
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
elif getattr(field, '__prevent_search__', False):
raise PermissionDenied(_('Filtering on %s is not allowed.' % name))
if field in field_list:
# Field traversed twice, could create infinite JOINs, DoSing Tower
raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name))
field_list.append(field)
model = getattr(field, 'related_model', None)
return field_list, '__'.join(new_parts)
def get_field_from_path(model, path):
"""
Given a Django ORM lookup path (possibly over multiple models)
Returns the last field in the line, and the revised lookup path
ex.
(<IntegerField for timeout>, 'project__timeout')
"""
field_list, new_path = get_fields_from_path(model, path)
return (field_list[-1], new_path)
class FieldLookupBackend(BaseFilterBackend):
"""
Filter using field lookups provided via query string parameters.
"""
RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate', 'limit')
SUPPORTED_LOOKUPS = (
'exact',
'iexact',
'contains',
'icontains',
'startswith',
'istartswith',
'endswith',
'iendswith',
'regex',
'iregex',
'gt',
'gte',
'lt',
'lte',
'in',
'isnull',
'search',
)
# A list of fields that we know can be filtered on without the possibility
# of introducing duplicates
NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField)
def get_fields_from_lookup(self, model, lookup):
if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS:
path, suffix = lookup.rsplit('__', 1)
else:
path = lookup
suffix = 'exact'
if not path:
raise ParseError(_('Query string field name not provided.'))
# FIXME: Could build up a list of models used across relationships, use
# those lookups combined with request.user.get_queryset(Model) to make
# sure user cannot query using objects he could not view.
field_list, new_path = get_fields_from_path(model, path)
new_lookup = new_path
new_lookup = '__'.join([new_path, suffix])
return field_list, new_lookup
def get_field_from_lookup(self, model, lookup):
'''Method to match return type of single field, if needed.'''
field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
return (field_list[-1], new_lookup)
def to_python_related(self, value):
value = force_str(value)
if value.lower() in ('none', 'null'):
return None
else:
return int(value)
def value_to_python_for_field(self, field, value):
if isinstance(field, models.BooleanField):
return to_python_boolean(value)
elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)):
try:
return self.to_python_related(value)
except ValueError:
raise ParseError(_('Invalid {field_name} id: {field_id}').format(field_name=getattr(field, 'name', 'related field'), field_id=value))
else:
return field.to_python(value)
def value_to_python(self, model, lookup, value):
try:
lookup.encode("ascii")
except UnicodeEncodeError:
raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup)
field_list, new_lookup = self.get_fields_from_lookup(model, lookup)
field = field_list[-1]
needs_distinct = not all(isinstance(f, self.NO_DUPLICATES_ALLOW_LIST) for f in field_list)
# Type names are stored without underscores internally, but are presented and
# and serialized over the API containing underscores so we remove `_`
# for polymorphic_ctype__model lookups.
if new_lookup.startswith('polymorphic_ctype__model'):
value = value.replace('_', '')
elif new_lookup.endswith('__isnull'):
value = to_python_boolean(value)
elif new_lookup.endswith('__in'):
items = []
if not value:
raise ValueError('cannot provide empty value for __in')
for item in value.split(','):
items.append(self.value_to_python_for_field(field, item))
value = items
elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'):
try:
re.compile(value)
except re.error as e:
raise ValueError(e.args[0])
elif new_lookup.endswith('__iexact'):
if not isinstance(field, (CharField, TextField)):
raise ValueError(f'{field.name} is not a text field and cannot be filtered by case-insensitive search')
elif new_lookup.endswith('__search'):
related_model = getattr(field, 'related_model', None)
if not related_model:
raise ValueError('%s is not searchable' % new_lookup[:-8])
new_lookups = []
for rm_field in related_model._meta.fields:
if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'):
new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name))
return value, new_lookups, needs_distinct
else:
if isinstance(field, JSONField):
new_lookup = new_lookup.replace(field.name, f'{field.name}_as_txt')
value = self.value_to_python_for_field(field, value)
return value, new_lookup, needs_distinct
def filter_queryset(self, request, queryset, view):
try:
# Apply filters specified via query_params. Each entry in the lists
# below is (negate, field, value).
and_filters = []
or_filters = []
chain_filters = []
role_filters = []
search_filters = {}
needs_distinct = False
# Can only have two values: 'AND', 'OR'
# If 'AND' is used, an item must satisfy all conditions to show up in the results.
# If 'OR' is used, an item just needs to satisfy one condition to appear in results.
search_filter_relation = 'OR'
for key, values in request.query_params.lists():
if key in self.RESERVED_NAMES:
continue
# HACK: make `created` available via API for the Django User ORM model
# so it keep compatibility with other objects which exposes the `created` attr.
if queryset.model._meta.object_name == 'User' and key.startswith('created'):
key = key.replace('created', 'date_joined')
# HACK: Make job event filtering by host name mostly work even
# when not capturing job event hosts M2M.
if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'):
key = key.replace('hosts__name', 'or__host__name')
or_filters.append((False, 'host__name__isnull', True))
# Custom __int filter suffix (internal use only).
q_int = False
if key.endswith('__int'):
key = key[:-5]
q_int = True
# RBAC filtering
if key == 'role_level':
role_filters.append(values[0])
continue
# Search across related objects.
if key.endswith('__search'):
if values and ',' in values[0]:
search_filter_relation = 'AND'
values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values])
for value in values:
search_value, new_keys, _ = self.value_to_python(queryset.model, key, force_str(value))
assert isinstance(new_keys, list)
search_filters[search_value] = new_keys
# by definition, search *only* joins across relations,
# so it _always_ needs a .distinct()
needs_distinct = True
continue
# Custom chain__ and or__ filters, mutually exclusive (both can
# precede not__).
q_chain = False
q_or = False
if key.startswith('chain__'):
key = key[7:]
q_chain = True
elif key.startswith('or__'):
key = key[4:]
q_or = True
# Custom not__ filter prefix.
q_not = False
if key.startswith('not__'):
key = key[5:]
q_not = True
# Convert value(s) to python and add to the appropriate list.
for value in values:
if q_int:
value = int(value)
value, new_key, distinct = self.value_to_python(queryset.model, key, value)
if distinct:
needs_distinct = True
if '_as_txt' in new_key:
fname = next(item for item in new_key.split('__') if item.endswith('_as_txt'))
queryset = queryset.annotate(**{fname: Cast(fname[:-7], output_field=TextField())})
if q_chain:
chain_filters.append((q_not, new_key, value))
elif q_or:
or_filters.append((q_not, new_key, value))
else:
and_filters.append((q_not, new_key, value))
# Now build Q objects for database query filter.
if and_filters or or_filters or chain_filters or role_filters or search_filters:
args = []
for n, k, v in and_filters:
if n:
args.append(~Q(**{k: v}))
else:
args.append(Q(**{k: v}))
for role_name in role_filters:
if not hasattr(queryset.model, 'accessible_pk_qs'):
raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.'))
args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name)))
if or_filters:
q = Q()
for n, k, v in or_filters:
if n:
q |= ~Q(**{k: v})
else:
q |= Q(**{k: v})
args.append(q)
if search_filters and search_filter_relation == 'OR':
q = Q()
for term, constrains in search_filters.items():
for constrain in constrains:
q |= Q(**{constrain: term})
args.append(q)
elif search_filters and search_filter_relation == 'AND':
for term, constrains in search_filters.items():
q_chain = Q()
for constrain in constrains:
q_chain |= Q(**{constrain: term})
queryset = queryset.filter(q_chain)
for n, k, v in chain_filters:
if n:
q = ~Q(**{k: v})
else:
q = Q(**{k: v})
queryset = queryset.filter(q)
queryset = queryset.filter(*args)
if needs_distinct:
queryset = queryset.distinct()
return queryset
except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e:
raise ParseError(e.args[0])
except ValidationError as e:
raise ParseError(json.dumps(e.messages, ensure_ascii=False))
class OrderByBackend(BaseFilterBackend):
"""
Filter to apply ordering based on query string parameters.
"""
def filter_queryset(self, request, queryset, view):
try:
order_by = None
for key, value in request.query_params.items():
if key in ('order', 'order_by'):
order_by = value
if ',' in value:
order_by = value.split(',')
else:
order_by = (value,)
default_order_by = self.get_default_ordering(view)
# glue the order by and default order by together so that the default is the backup option
order_by = list(order_by or []) + list(default_order_by or [])
if order_by:
order_by = self._validate_ordering_fields(queryset.model, order_by)
# Special handling of the type field for ordering. In this
# case, we're not sorting exactly on the type field, but
# given the limited number of views with multiple types,
# sorting on polymorphic_ctype.model is effectively the same.
new_order_by = []
if 'polymorphic_ctype' in get_all_field_names(queryset.model):
for field in order_by:
if field == 'type':
new_order_by.append('polymorphic_ctype__model')
elif field == '-type':
new_order_by.append('-polymorphic_ctype__model')
else:
new_order_by.append(field)
else:
for field in order_by:
if field not in ('type', '-type'):
new_order_by.append(field)
queryset = queryset.order_by(*new_order_by)
return queryset
except FieldError as e:
# Return a 400 for invalid field names.
raise ParseError(*e.args)
def get_default_ordering(self, view):
ordering = getattr(view, 'ordering', None)
if isinstance(ordering, str):
return (ordering,)
return ordering
def _validate_ordering_fields(self, model, order_by):
for field_name in order_by:
# strip off the negation prefix `-` if it exists
prefix = ''
path = field_name
if field_name[0] == '-':
prefix = field_name[0]
path = field_name[1:]
try:
field, new_path = get_field_from_path(model, path)
new_path = '{}{}'.format(prefix, new_path)
except (FieldError, FieldDoesNotExist) as e:
raise ParseError(e.args[0])
yield new_path

5
awx/api/generics.py
View File

@@ -30,12 +30,13 @@ from rest_framework.permissions import IsAuthenticated
from rest_framework.renderers import StaticHTMLRenderer from rest_framework.renderers import StaticHTMLRenderer
from rest_framework.negotiation import DefaultContentNegotiation from rest_framework.negotiation import DefaultContentNegotiation
from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
from ansible_base.utils.models import get_all_field_names
# AWX # AWX
from awx.api.filters import FieldLookupBackend
from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate
from awx.main.access import optimize_queryset from awx.main.access import optimize_queryset
from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version
from awx.main.utils.db import get_all_field_names
from awx.main.utils.licensing import server_product_name from awx.main.utils.licensing import server_product_name
from awx.main.views import ApiErrorView from awx.main.views import ApiErrorView
from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer

3
awx/api/serializers.py
View File

@@ -43,6 +43,8 @@ from rest_framework.utils.serializer_helpers import ReturnList
# Django-Polymorphic # Django-Polymorphic
from polymorphic.models import PolymorphicModel from polymorphic.models import PolymorphicModel
from ansible_base.utils.models import get_type_for_model
# AWX # AWX
from awx.main.access import get_user_capabilities from awx.main.access import get_user_capabilities
from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE
@@ -102,7 +104,6 @@ from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES
from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry
from awx.main.fields import ImplicitRoleField from awx.main.fields import ImplicitRoleField
from awx.main.utils import ( from awx.main.utils import (
get_type_for_model,
get_model_for_type, get_model_for_type,
camelcase_to_underscore, camelcase_to_underscore,
getattrd, getattrd,

4
awx/conf/models.py
View File

@@ -7,8 +7,10 @@ import json
# Django # Django
from django.db import models from django.db import models
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.main.models.base import CreatedModifiedModel, prevent_search from awx.main.models.base import CreatedModifiedModel
from awx.main.utils import encrypt_field from awx.main.utils import encrypt_field
from awx.conf import settings_registry from awx.conf import settings_registry

3
awx/main/access.py
View File

@@ -20,11 +20,12 @@ from rest_framework.exceptions import ParseError, PermissionDenied
# Django OAuth Toolkit # Django OAuth Toolkit
from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken
from ansible_base.utils.validation import to_python_boolean
# AWX # AWX
from awx.main.utils import ( from awx.main.utils import (
get_object_or_400, get_object_or_400,
get_pk_from_dict, get_pk_from_dict,
to_python_boolean,
get_licenser, get_licenser,
) )
from awx.main.models import ( from awx.main.models import (

4
awx/main/models/__init__.py
View File

@@ -6,8 +6,10 @@ from django.conf import settings # noqa
from django.db import connection from django.db import connection
from django.db.models.signals import pre_delete # noqa from django.db.models.signals import pre_delete # noqa
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa
from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa
from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa

4
awx/main/models/ad_hoc_commands.py
View File

@@ -12,9 +12,11 @@ from django.utils.text import Truncator
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from django.core.exceptions import ValidationError from django.core.exceptions import ValidationError
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty from awx.main.models.base import AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty
from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent
from awx.main.models.unified_jobs import UnifiedJob from awx.main.models.unified_jobs import UnifiedJob
from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate

18
awx/main/models/base.py
View File

@@ -15,7 +15,6 @@ from awx.main.utils import encrypt_field, parse_yaml_or_json
from awx.main.constants import CLOUD_PROVIDERS from awx.main.constants import CLOUD_PROVIDERS
__all__ = [ __all__ = [
'prevent_search',
'VarsDictProperty', 'VarsDictProperty',
'BaseModel', 'BaseModel',
'CreatedModifiedModel', 'CreatedModifiedModel',
@@ -384,23 +383,6 @@ class NotificationFieldsModel(BaseModel):
notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started') notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started')
def prevent_search(relation):
"""
Used to mark a model field or relation as "restricted from filtering"
e.g.,
class AuthToken(BaseModel):
user = prevent_search(models.ForeignKey(...))
sensitive_data = prevent_search(models.CharField(...))
The flag set by this function is used by
`awx.api.filters.FieldLookupBackend` to block fields and relations that
should not be searchable/filterable via search query params
"""
setattr(relation, '__prevent_search__', True)
return relation
def accepts_json(relation): def accepts_json(relation):
""" """
Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars

4
awx/main/models/ha.py
View File

@@ -17,6 +17,8 @@ from django.db.models import Sum, Q
import redis import redis
from solo.models import SingletonModel from solo.models import SingletonModel
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx import __version__ as awx_application_version from awx import __version__ as awx_application_version
from awx.main.utils import is_testing from awx.main.utils import is_testing
@@ -24,7 +26,7 @@ from awx.api.versioning import reverse
from awx.main.fields import ImplicitRoleField from awx.main.fields import ImplicitRoleField
from awx.main.managers import InstanceManager, UUID_DEFAULT from awx.main.managers import InstanceManager, UUID_DEFAULT
from awx.main.constants import JOB_FOLDER_PREFIX from awx.main.constants import JOB_FOLDER_PREFIX
from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search from awx.main.models.base import BaseModel, HasEditsMixin
from awx.main.models.rbac import ( from awx.main.models.rbac import (
ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_ADMINISTRATOR,
ROLE_SINGLETON_SYSTEM_AUDITOR, ROLE_SINGLETON_SYSTEM_AUDITOR,

4
awx/main/models/inventory.py
View File

@@ -25,6 +25,8 @@ from django.db.models import Q
# REST Framework # REST Framework
from rest_framework.exceptions import ParseError from rest_framework.exceptions import ParseError
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.constants import CLOUD_PROVIDERS from awx.main.constants import CLOUD_PROVIDERS
@@ -35,7 +37,7 @@ from awx.main.fields import (
OrderedManyToManyField, OrderedManyToManyField,
) )
from awx.main.managers import HostManager, HostMetricActiveManager from awx.main.managers import HostManager, HostMetricActiveManager
from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json
from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent
from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate
from awx.main.models.mixins import ( from awx.main.models.mixins import (

3
awx/main/models/jobs.py
View File

@@ -20,13 +20,14 @@ from django.core.exceptions import FieldDoesNotExist
# REST Framework # REST Framework
from rest_framework.exceptions import ParseError from rest_framework.exceptions import ParseError
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.constants import HOST_FACTS_FIELDS from awx.main.constants import HOST_FACTS_FIELDS
from awx.main.models.base import ( from awx.main.models.base import (
BaseModel, BaseModel,
CreatedModifiedModel, CreatedModifiedModel,
prevent_search,
accepts_json, accepts_json,
JOB_TYPE_CHOICES, JOB_TYPE_CHOICES,
NEW_JOB_TYPE_CHOICES, NEW_JOB_TYPE_CHOICES,

3
awx/main/models/mixins.py
View File

@@ -16,8 +16,9 @@ from django.db.models.query import QuerySet
from django.utils.crypto import get_random_string from django.utils.crypto import get_random_string
from django.utils.translation import gettext_lazy as _ from django.utils.translation import gettext_lazy as _
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.main.models.base import prevent_search
from awx.main.models.rbac import Role, RoleAncestorEntry from awx.main.models.rbac import Role, RoleAncestorEntry
from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
from awx.main.utils.execution_environments import get_default_execution_environment from awx.main.utils.execution_environments import get_default_execution_environment

4
awx/main/models/notifications.py
View File

@@ -15,9 +15,11 @@ from django.utils.encoding import smart_str, force_str
from jinja2 import sandbox, ChainableUndefined from jinja2 import sandbox, ChainableUndefined
from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel
from awx.main.utils import encrypt_field, decrypt_field, set_environ from awx.main.utils import encrypt_field, decrypt_field, set_environ
from awx.main.notifications.email_backend import CustomEmailBackend from awx.main.notifications.email_backend import CustomEmailBackend
from awx.main.notifications.slack_backend import SlackBackend from awx.main.notifications.slack_backend import SlackBackend

5
awx/main/models/unified_jobs.py
View File

@@ -30,8 +30,10 @@ from rest_framework.exceptions import ParseError
# Django-Polymorphic # Django-Polymorphic
from polymorphic.models import PolymorphicModel from polymorphic.models import PolymorphicModel
from ansible_base.utils.models import prevent_search, get_type_for_model
# AWX # AWX
from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel
from awx.main.dispatch import get_task_queuename from awx.main.dispatch import get_task_queuename
from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.dispatch.control import Control as ControlDispatcher
from awx.main.registrar import activity_stream_registrar from awx.main.registrar import activity_stream_registrar
@@ -42,7 +44,6 @@ from awx.main.utils.common import (
_inventory_updates, _inventory_updates,
copy_model_by_class, copy_model_by_class,
copy_m2m_relationships, copy_m2m_relationships,
get_type_for_model,
parse_yaml_or_json, parse_yaml_or_json,
getattr_dne, getattr_dne,
ScheduleDependencyManager, ScheduleDependencyManager,

4
awx/main/models/workflow.py
View File

@@ -23,9 +23,11 @@ from crum import get_current_user
from jinja2 import sandbox from jinja2 import sandbox
from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError
from ansible_base.utils.models import prevent_search
# AWX # AWX
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob from awx.main.models import accepts_json, UnifiedJobTemplate, UnifiedJob
from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin
from awx.main.models.base import CreatedModifiedModel, VarsDictProperty from awx.main.models.base import CreatedModifiedModel, VarsDictProperty
from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR

3
awx/main/scheduler/task_manager.py
View File

@@ -17,6 +17,8 @@ from django.utils.timezone import now as tz_now
from django.conf import settings from django.conf import settings
from django.contrib.contenttypes.models import ContentType from django.contrib.contenttypes.models import ContentType
from ansible_base.utils.models import get_type_for_model
# AWX # AWX
from awx.main.dispatch.reaper import reap_job from awx.main.dispatch.reaper import reap_job
from awx.main.models import ( from awx.main.models import (
@@ -34,7 +36,6 @@ from awx.main.models import (
from awx.main.scheduler.dag_workflow import WorkflowDAG from awx.main.scheduler.dag_workflow import WorkflowDAG
from awx.main.utils.pglock import advisory_lock from awx.main.utils.pglock import advisory_lock
from awx.main.utils import ( from awx.main.utils import (
get_type_for_model,
ScheduleTaskManager, ScheduleTaskManager,
ScheduleWorkflowManager, ScheduleWorkflowManager,
) )

2
awx/main/tests/functional/api/test_survey_spec.py
View File

@@ -2,12 +2,12 @@ from unittest import mock
import pytest import pytest
import json import json
from ansible_base.utils.models import get_type_for_model
from awx.api.versioning import reverse from awx.api.versioning import reverse
from awx.main.models.jobs import JobTemplate, Job from awx.main.models.jobs import JobTemplate, Job
from awx.main.models.activity_stream import ActivityStream from awx.main.models.activity_stream import ActivityStream
from awx.main.access import JobTemplateAccess from awx.main.access import JobTemplateAccess
from awx.main.utils.common import get_type_for_model
@pytest.fixture @pytest.fixture

90
awx/main/tests/unit/api/test_filters.py
View File

@@ -3,15 +3,13 @@
import pytest import pytest
# Django # Django
from django.core.exceptions import FieldDoesNotExist from rest_framework.exceptions import PermissionDenied
from rest_framework.exceptions import PermissionDenied, ParseError from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path
from awx.main.models import ( from awx.main.models import (
AdHocCommand, AdHocCommand,
ActivityStream, ActivityStream,
Credential,
Job, Job,
JobTemplate, JobTemplate,
SystemJob, SystemJob,
@@ -20,88 +18,11 @@ from awx.main.models import (
WorkflowJob, WorkflowJob,
WorkflowJobTemplate, WorkflowJobTemplate,
WorkflowJobOptions, WorkflowJobOptions,
InventorySource,
JobEvent,
) )
from awx.main.models.oauth import OAuth2Application from awx.main.models.oauth import OAuth2Application
from awx.main.models.jobs import JobOptions from awx.main.models.jobs import JobOptions
def test_related():
field_lookup = FieldLookupBackend()
lookup = '__'.join(['inventory', 'organization', 'pk'])
field, new_lookup = field_lookup.get_field_from_lookup(InventorySource, lookup)
print(field)
print(new_lookup)
def test_invalid_filter_key():
field_lookup = FieldLookupBackend()
# FieldDoesNotExist is caught and converted to ParseError by filter_queryset
with pytest.raises(FieldDoesNotExist) as excinfo:
field_lookup.value_to_python(JobEvent, 'event_data.task_action', 'foo')
assert 'has no field named' in str(excinfo)
def test_invalid_field_hop():
with pytest.raises(ParseError) as excinfo:
get_field_from_path(Credential, 'organization__description__user')
assert 'No related model for' in str(excinfo)
def test_invalid_order_by_key():
field_order_by = OrderByBackend()
with pytest.raises(ParseError) as excinfo:
[f for f in field_order_by._validate_ordering_fields(JobEvent, ('event_data.task_action',))]
assert 'has no field named' in str(excinfo)
@pytest.mark.parametrize(u"empty_value", [u'', ''])
def test_empty_in(empty_value):
field_lookup = FieldLookupBackend()
with pytest.raises(ValueError) as excinfo:
field_lookup.value_to_python(JobTemplate, 'project__name__in', empty_value)
assert 'empty value for __in' in str(excinfo.value)
@pytest.mark.parametrize(u"valid_value", [u'foo', u'foo,'])
def test_valid_in(valid_value):
field_lookup = FieldLookupBackend()
value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__in', valid_value)
assert 'foo' in value
def test_invalid_field():
invalid_field = u"ヽヾ"
field_lookup = FieldLookupBackend()
with pytest.raises(ValueError) as excinfo:
field_lookup.value_to_python(WorkflowJobTemplate, invalid_field, 'foo')
assert 'is not an allowed field name. Must be ascii encodable.' in str(excinfo.value)
def test_valid_iexact():
field_lookup = FieldLookupBackend()
value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__iexact', 'foo')
assert 'foo' in value
def test_invalid_iexact():
field_lookup = FieldLookupBackend()
with pytest.raises(ValueError) as excinfo:
field_lookup.value_to_python(Job, 'id__iexact', '1')
assert 'is not a text field and cannot be filtered by case-insensitive search' in str(excinfo.value)
@pytest.mark.parametrize('lookup_suffix', ['', 'contains', 'startswith', 'in'])
@pytest.mark.parametrize('password_field', Credential.PASSWORD_FIELDS)
def test_filter_on_password_field(password_field, lookup_suffix):
field_lookup = FieldLookupBackend()
lookup = '__'.join(filter(None, [password_field, lookup_suffix]))
with pytest.raises(PermissionDenied) as excinfo:
field, new_lookup = field_lookup.get_field_from_lookup(Credential, lookup)
assert 'not allowed' in str(excinfo.value)
@pytest.mark.parametrize( @pytest.mark.parametrize(
'model, query', 'model, query',
[ [
@@ -128,10 +49,3 @@ def test_filter_sensitive_fields_and_relations(model, query):
with pytest.raises(PermissionDenied) as excinfo: with pytest.raises(PermissionDenied) as excinfo:
field, new_lookup = field_lookup.get_field_from_lookup(model, query) field, new_lookup = field_lookup.get_field_from_lookup(model, query)
assert 'not allowed' in str(excinfo.value) assert 'not allowed' in str(excinfo.value)
def test_looping_filters_prohibited():
field_lookup = FieldLookupBackend()
with pytest.raises(ParseError) as loop_exc:
field_lookup.get_field_from_lookup(Job, 'job_events__job__job_events')
assert 'job_events' in str(loop_exc.value)

4
awx/main/tests/unit/utils/test_common.py
View File

@@ -12,6 +12,8 @@ from unittest import mock
from rest_framework.exceptions import ParseError from rest_framework.exceptions import ParseError
from ansible_base.utils.models import get_type_for_model
from awx.main.utils import common from awx.main.utils import common
from awx.api.validators import HostnameRegexValidator from awx.api.validators import HostnameRegexValidator
@@ -106,7 +108,7 @@ TEST_MODELS = [
# Cases relied on for scheduler dependent jobs list # Cases relied on for scheduler dependent jobs list
@pytest.mark.parametrize('model,name', TEST_MODELS) @pytest.mark.parametrize('model,name', TEST_MODELS)
def test_get_type_for_model(model, name): def test_get_type_for_model(model, name):
assert common.get_type_for_model(model) == name assert get_type_for_model(model) == name
def test_get_model_for_invalid_type(): def test_get_model_for_invalid_type():

4
awx/main/tests/unit/utils/test_filters.py
View File

@@ -68,7 +68,9 @@ class mockHost:
@mock.patch('awx.main.utils.filters.get_model', return_value=mockHost()) @mock.patch('awx.main.utils.filters.get_model', return_value=mockHost())
class TestSmartFilterQueryFromString: class TestSmartFilterQueryFromString:
@mock.patch('awx.api.filters.get_fields_from_path', lambda model, path: ([model], path)) # disable field filtering, because a__b isn't a real Host field @mock.patch(
'ansible_base.filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path: ([model], path)
) # disable field filtering, because a__b isn't a real Host field
@pytest.mark.parametrize( @pytest.mark.parametrize(
"filter_string,q_expected", "filter_string,q_expected",
[ [

22
awx/main/utils/common.py
View File

@@ -52,12 +52,10 @@ __all__ = [
'get_awx_http_client_headers', 'get_awx_http_client_headers',
'get_awx_version', 'get_awx_version',
'update_scm_url', 'update_scm_url',
'get_type_for_model',
'get_model_for_type', 'get_model_for_type',
'copy_model_by_class', 'copy_model_by_class',
'copy_m2m_relationships', 'copy_m2m_relationships',
'prefetch_page_capabilities', 'prefetch_page_capabilities',
'to_python_boolean',
'datetime_hook', 'datetime_hook',
'ignore_inventory_computed_fields', 'ignore_inventory_computed_fields',
'ignore_inventory_group_removal', 'ignore_inventory_group_removal',
@@ -110,18 +108,6 @@ def get_object_or_400(klass, *args, **kwargs):
raise ParseError(*e.args) raise ParseError(*e.args)
def to_python_boolean(value, allow_none=False):
value = str(value)
if value.lower() in ('true', '1', 't'):
return True
elif value.lower() in ('false', '0', 'f'):
return False
elif allow_none and value.lower() in ('none', 'null'):
return None
else:
raise ValueError(_(u'Unable to convert "%s" to boolean') % value)
def datetime_hook(d): def datetime_hook(d):
new_d = {} new_d = {}
for key, value in d.items(): for key, value in d.items():
@@ -569,14 +555,6 @@ def copy_m2m_relationships(obj1, obj2, fields, kwargs=None):
dest_field.add(*list(src_field_value.all().values_list('id', flat=True))) dest_field.add(*list(src_field_value.all().values_list('id', flat=True)))
def get_type_for_model(model):
"""
Return type name for a given model class.
"""
opts = model._meta.concrete_model._meta
return camelcase_to_underscore(opts.object_name)
def get_model_for_type(type_name): def get_model_for_type(type_name):
""" """
Return model class for a given type name. Return model class for a given type name.

17
awx/main/utils/db.py
View File

@@ -1,27 +1,10 @@
# Copyright (c) 2017 Ansible by Red Hat # Copyright (c) 2017 Ansible by Red Hat
# All Rights Reserved. # All Rights Reserved.
from itertools import chain
from awx.settings.application_name import set_application_name from awx.settings.application_name import set_application_name
from django.conf import settings from django.conf import settings
def get_all_field_names(model):
# Implements compatibility with _meta.get_all_field_names
# See: https://docs.djangoproject.com/en/1.11/ref/models/meta/#migrating-from-the-old-api
return list(
set(
chain.from_iterable(
(field.name, field.attname) if hasattr(field, 'attname') else (field.name,)
for field in model._meta.get_fields()
# For complete backwards compatibility, you may want to exclude
# GenericForeignKey from the results.
if not (field.many_to_one and field.related_model is None)
)
)
)
def set_connection_name(function): def set_connection_name(function):
set_application_name(settings.DATABASES, settings.CLUSTER_HOST_ID, function=function) set_application_name(settings.DATABASES, settings.CLUSTER_HOST_ID, function=function)

2
awx/main/utils/filters.py
View File

@@ -161,7 +161,7 @@ class SmartFilter(object):
else: else:
# detect loops and restrict access to sensitive fields # detect loops and restrict access to sensitive fields
# this import is intentional here to avoid a circular import # this import is intentional here to avoid a circular import
from awx.api.filters import FieldLookupBackend from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend
FieldLookupBackend().get_field_from_lookup(Host, k) FieldLookupBackend().get_field_from_lookup(Host, k)
kwargs[k] = v kwargs[k] = v

16
awx/settings/defaults.py
View File

@@ -11,6 +11,7 @@ from datetime import timedelta
# python-ldap # python-ldap
import ldap import ldap
from split_settings.tools import include
DEBUG = True DEBUG = True
@@ -354,12 +355,6 @@ REST_FRAMEWORK = {
'awx.api.authentication.LoggedBasicAuthentication', 'awx.api.authentication.LoggedBasicAuthentication',
), ),
'DEFAULT_PERMISSION_CLASSES': ('awx.api.permissions.ModelAccessPermission',), 'DEFAULT_PERMISSION_CLASSES': ('awx.api.permissions.ModelAccessPermission',),
'DEFAULT_FILTER_BACKENDS': (
'awx.api.filters.TypeFilterBackend',
'awx.api.filters.FieldLookupBackend',
'rest_framework.filters.SearchFilter',
'awx.api.filters.OrderByBackend',
),
'DEFAULT_PARSER_CLASSES': ('awx.api.parsers.JSONParser',), 'DEFAULT_PARSER_CLASSES': ('awx.api.parsers.JSONParser',),
'DEFAULT_RENDERER_CLASSES': ('awx.api.renderers.DefaultJSONRenderer', 'awx.api.renderers.BrowsableAPIRenderer'), 'DEFAULT_RENDERER_CLASSES': ('awx.api.renderers.DefaultJSONRenderer', 'awx.api.renderers.BrowsableAPIRenderer'),
'DEFAULT_METADATA_CLASS': 'awx.api.metadata.Metadata', 'DEFAULT_METADATA_CLASS': 'awx.api.metadata.Metadata',
@@ -1067,3 +1062,12 @@ CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36 # months
# Host metric summary monthly task - last time of run # Host metric summary monthly task - last time of run
HOST_METRIC_SUMMARY_TASK_LAST_TS = None HOST_METRIC_SUMMARY_TASK_LAST_TS = None
HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days
# django-ansible-base
ANSIBLE_BASE_FEATURES = {'AUTHENTICATION': False, 'SWAGGER': False, 'FILTERING': True}
from ansible_base import settings # noqa: E402
settings_file = os.path.join(os.path.dirname(settings.__file__), 'dynamic_settings.py')
include(settings_file)