From aacf9653c590e7f4ad563af7d58892bd3befba61 Mon Sep 17 00:00:00 2001 From: John Westcott IV <32551173+john-westcott-iv@users.noreply.github.com> Date: Mon, 18 Dec 2023 10:05:02 -0500 Subject: [PATCH] Use filtering/sorting from django-ansible-base (#14726) * Move filtering to DAB * add comment to trigger building a new image Signed-off-by: jessicamack * remove unneeded comment Signed-off-by: jessicamack * remove unused imports Signed-off-by: jessicamack * change mock import Signed-off-by: jessicamack --------- Signed-off-by: jessicamack Co-authored-by: jessicamack --- awx/api/filters.py | 450 ------------------ awx/api/generics.py | 5 +- awx/api/serializers.py | 3 +- awx/conf/models.py | 4 +- awx/main/access.py | 3 +- awx/main/models/__init__.py | 4 +- awx/main/models/ad_hoc_commands.py | 4 +- awx/main/models/base.py | 18 - awx/main/models/ha.py | 4 +- awx/main/models/inventory.py | 4 +- awx/main/models/jobs.py | 3 +- awx/main/models/mixins.py | 3 +- awx/main/models/notifications.py | 4 +- awx/main/models/unified_jobs.py | 5 +- awx/main/models/workflow.py | 4 +- awx/main/scheduler/task_manager.py | 3 +- .../tests/functional/api/test_survey_spec.py | 2 +- awx/main/tests/unit/api/test_filters.py | 90 +--- awx/main/tests/unit/utils/test_common.py | 4 +- awx/main/tests/unit/utils/test_filters.py | 4 +- awx/main/utils/common.py | 22 - awx/main/utils/db.py | 17 - awx/main/utils/filters.py | 2 +- awx/settings/defaults.py | 16 +- 24 files changed, 57 insertions(+), 621 deletions(-) delete mode 100644 awx/api/filters.py diff --git a/awx/api/filters.py b/awx/api/filters.py deleted file mode 100644 index 6169dc548a..0000000000 --- a/awx/api/filters.py +++ /dev/null @@ -1,450 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved. - -# Python -import re -import json -from functools import reduce - -# Django -from django.core.exceptions import FieldError, ValidationError, FieldDoesNotExist -from django.db import models -from django.db.models import Q, CharField, IntegerField, BooleanField, TextField, JSONField -from django.db.models.fields.related import ForeignObjectRel, ManyToManyField, ForeignKey -from django.db.models.functions import Cast -from django.contrib.contenttypes.models import ContentType -from django.contrib.contenttypes.fields import GenericForeignKey -from django.utils.encoding import force_str -from django.utils.translation import gettext_lazy as _ - -# Django REST Framework -from rest_framework.exceptions import ParseError, PermissionDenied -from rest_framework.filters import BaseFilterBackend - -# AWX -from awx.main.utils import get_type_for_model, to_python_boolean -from awx.main.utils.db import get_all_field_names - - -class TypeFilterBackend(BaseFilterBackend): - """ - Filter on type field now returned with all objects. - """ - - def filter_queryset(self, request, queryset, view): - try: - types = None - for key, value in request.query_params.items(): - if key == 'type': - if ',' in value: - types = value.split(',') - else: - types = (value,) - if types: - types_map = {} - for ct in ContentType.objects.filter(Q(app_label='main') | Q(app_label='auth', model='user')): - ct_model = ct.model_class() - if not ct_model: - continue - ct_type = get_type_for_model(ct_model) - types_map[ct_type] = ct.pk - model = queryset.model - model_type = get_type_for_model(model) - if 'polymorphic_ctype' in get_all_field_names(model): - types_pks = set([v for k, v in types_map.items() if k in types]) - queryset = queryset.filter(polymorphic_ctype_id__in=types_pks) - elif model_type in types: - queryset = queryset - else: - queryset = queryset.none() - return queryset - except FieldError as e: - # Return a 400 for invalid field names. - raise ParseError(*e.args) - - -def get_fields_from_path(model, path): - """ - Given a Django ORM lookup path (possibly over multiple models) - Returns the fields in the line, and also the revised lookup path - ex., given - model=Organization - path='project__timeout' - returns tuple of fields traversed as well and a corrected path, - for special cases we do substitutions - ([], 'project__timeout') - """ - # Store of all the fields used to detect repeats - field_list = [] - new_parts = [] - for name in path.split('__'): - if model is None: - raise ParseError(_('No related model for field {}.').format(name)) - # HACK: Make project and inventory source filtering by old field names work for backwards compatibility. - if model._meta.object_name in ('Project', 'InventorySource'): - name = {'current_update': 'current_job', 'last_update': 'last_job', 'last_update_failed': 'last_job_failed', 'last_updated': 'last_job_run'}.get( - name, name - ) - - if name == 'type' and 'polymorphic_ctype' in get_all_field_names(model): - name = 'polymorphic_ctype' - new_parts.append('polymorphic_ctype__model') - else: - new_parts.append(name) - - if name in getattr(model, 'PASSWORD_FIELDS', ()): - raise PermissionDenied(_('Filtering on password fields is not allowed.')) - elif name == 'pk': - field = model._meta.pk - else: - name_alt = name.replace("_", "") - if name_alt in model._meta.fields_map.keys(): - field = model._meta.fields_map[name_alt] - new_parts.pop() - new_parts.append(name_alt) - else: - field = model._meta.get_field(name) - if isinstance(field, ForeignObjectRel) and getattr(field.field, '__prevent_search__', False): - raise PermissionDenied(_('Filtering on %s is not allowed.' % name)) - elif getattr(field, '__prevent_search__', False): - raise PermissionDenied(_('Filtering on %s is not allowed.' % name)) - if field in field_list: - # Field traversed twice, could create infinite JOINs, DoSing Tower - raise ParseError(_('Loops not allowed in filters, detected on field {}.').format(field.name)) - field_list.append(field) - model = getattr(field, 'related_model', None) - - return field_list, '__'.join(new_parts) - - -def get_field_from_path(model, path): - """ - Given a Django ORM lookup path (possibly over multiple models) - Returns the last field in the line, and the revised lookup path - ex. - (, 'project__timeout') - """ - field_list, new_path = get_fields_from_path(model, path) - return (field_list[-1], new_path) - - -class FieldLookupBackend(BaseFilterBackend): - """ - Filter using field lookups provided via query string parameters. - """ - - RESERVED_NAMES = ('page', 'page_size', 'format', 'order', 'order_by', 'search', 'type', 'host_filter', 'count_disabled', 'no_truncate', 'limit') - - SUPPORTED_LOOKUPS = ( - 'exact', - 'iexact', - 'contains', - 'icontains', - 'startswith', - 'istartswith', - 'endswith', - 'iendswith', - 'regex', - 'iregex', - 'gt', - 'gte', - 'lt', - 'lte', - 'in', - 'isnull', - 'search', - ) - - # A list of fields that we know can be filtered on without the possibility - # of introducing duplicates - NO_DUPLICATES_ALLOW_LIST = (CharField, IntegerField, BooleanField, TextField) - - def get_fields_from_lookup(self, model, lookup): - if '__' in lookup and lookup.rsplit('__', 1)[-1] in self.SUPPORTED_LOOKUPS: - path, suffix = lookup.rsplit('__', 1) - else: - path = lookup - suffix = 'exact' - - if not path: - raise ParseError(_('Query string field name not provided.')) - - # FIXME: Could build up a list of models used across relationships, use - # those lookups combined with request.user.get_queryset(Model) to make - # sure user cannot query using objects he could not view. - field_list, new_path = get_fields_from_path(model, path) - - new_lookup = new_path - new_lookup = '__'.join([new_path, suffix]) - return field_list, new_lookup - - def get_field_from_lookup(self, model, lookup): - '''Method to match return type of single field, if needed.''' - field_list, new_lookup = self.get_fields_from_lookup(model, lookup) - return (field_list[-1], new_lookup) - - def to_python_related(self, value): - value = force_str(value) - if value.lower() in ('none', 'null'): - return None - else: - return int(value) - - def value_to_python_for_field(self, field, value): - if isinstance(field, models.BooleanField): - return to_python_boolean(value) - elif isinstance(field, (ForeignObjectRel, ManyToManyField, GenericForeignKey, ForeignKey)): - try: - return self.to_python_related(value) - except ValueError: - raise ParseError(_('Invalid {field_name} id: {field_id}').format(field_name=getattr(field, 'name', 'related field'), field_id=value)) - else: - return field.to_python(value) - - def value_to_python(self, model, lookup, value): - try: - lookup.encode("ascii") - except UnicodeEncodeError: - raise ValueError("%r is not an allowed field name. Must be ascii encodable." % lookup) - - field_list, new_lookup = self.get_fields_from_lookup(model, lookup) - field = field_list[-1] - - needs_distinct = not all(isinstance(f, self.NO_DUPLICATES_ALLOW_LIST) for f in field_list) - - # Type names are stored without underscores internally, but are presented and - # and serialized over the API containing underscores so we remove `_` - # for polymorphic_ctype__model lookups. - if new_lookup.startswith('polymorphic_ctype__model'): - value = value.replace('_', '') - elif new_lookup.endswith('__isnull'): - value = to_python_boolean(value) - elif new_lookup.endswith('__in'): - items = [] - if not value: - raise ValueError('cannot provide empty value for __in') - for item in value.split(','): - items.append(self.value_to_python_for_field(field, item)) - value = items - elif new_lookup.endswith('__regex') or new_lookup.endswith('__iregex'): - try: - re.compile(value) - except re.error as e: - raise ValueError(e.args[0]) - elif new_lookup.endswith('__iexact'): - if not isinstance(field, (CharField, TextField)): - raise ValueError(f'{field.name} is not a text field and cannot be filtered by case-insensitive search') - elif new_lookup.endswith('__search'): - related_model = getattr(field, 'related_model', None) - if not related_model: - raise ValueError('%s is not searchable' % new_lookup[:-8]) - new_lookups = [] - for rm_field in related_model._meta.fields: - if rm_field.name in ('username', 'first_name', 'last_name', 'email', 'name', 'description', 'playbook'): - new_lookups.append('{}__{}__icontains'.format(new_lookup[:-8], rm_field.name)) - return value, new_lookups, needs_distinct - else: - if isinstance(field, JSONField): - new_lookup = new_lookup.replace(field.name, f'{field.name}_as_txt') - value = self.value_to_python_for_field(field, value) - return value, new_lookup, needs_distinct - - def filter_queryset(self, request, queryset, view): - try: - # Apply filters specified via query_params. Each entry in the lists - # below is (negate, field, value). - and_filters = [] - or_filters = [] - chain_filters = [] - role_filters = [] - search_filters = {} - needs_distinct = False - # Can only have two values: 'AND', 'OR' - # If 'AND' is used, an item must satisfy all conditions to show up in the results. - # If 'OR' is used, an item just needs to satisfy one condition to appear in results. - search_filter_relation = 'OR' - for key, values in request.query_params.lists(): - if key in self.RESERVED_NAMES: - continue - - # HACK: make `created` available via API for the Django User ORM model - # so it keep compatibility with other objects which exposes the `created` attr. - if queryset.model._meta.object_name == 'User' and key.startswith('created'): - key = key.replace('created', 'date_joined') - - # HACK: Make job event filtering by host name mostly work even - # when not capturing job event hosts M2M. - if queryset.model._meta.object_name == 'JobEvent' and key.startswith('hosts__name'): - key = key.replace('hosts__name', 'or__host__name') - or_filters.append((False, 'host__name__isnull', True)) - - # Custom __int filter suffix (internal use only). - q_int = False - if key.endswith('__int'): - key = key[:-5] - q_int = True - - # RBAC filtering - if key == 'role_level': - role_filters.append(values[0]) - continue - - # Search across related objects. - if key.endswith('__search'): - if values and ',' in values[0]: - search_filter_relation = 'AND' - values = reduce(lambda list1, list2: list1 + list2, [i.split(',') for i in values]) - for value in values: - search_value, new_keys, _ = self.value_to_python(queryset.model, key, force_str(value)) - assert isinstance(new_keys, list) - search_filters[search_value] = new_keys - # by definition, search *only* joins across relations, - # so it _always_ needs a .distinct() - needs_distinct = True - continue - - # Custom chain__ and or__ filters, mutually exclusive (both can - # precede not__). - q_chain = False - q_or = False - if key.startswith('chain__'): - key = key[7:] - q_chain = True - elif key.startswith('or__'): - key = key[4:] - q_or = True - - # Custom not__ filter prefix. - q_not = False - if key.startswith('not__'): - key = key[5:] - q_not = True - - # Convert value(s) to python and add to the appropriate list. - for value in values: - if q_int: - value = int(value) - value, new_key, distinct = self.value_to_python(queryset.model, key, value) - if distinct: - needs_distinct = True - if '_as_txt' in new_key: - fname = next(item for item in new_key.split('__') if item.endswith('_as_txt')) - queryset = queryset.annotate(**{fname: Cast(fname[:-7], output_field=TextField())}) - if q_chain: - chain_filters.append((q_not, new_key, value)) - elif q_or: - or_filters.append((q_not, new_key, value)) - else: - and_filters.append((q_not, new_key, value)) - - # Now build Q objects for database query filter. - if and_filters or or_filters or chain_filters or role_filters or search_filters: - args = [] - for n, k, v in and_filters: - if n: - args.append(~Q(**{k: v})) - else: - args.append(Q(**{k: v})) - for role_name in role_filters: - if not hasattr(queryset.model, 'accessible_pk_qs'): - raise ParseError(_('Cannot apply role_level filter to this list because its model does not use roles for access control.')) - args.append(Q(pk__in=queryset.model.accessible_pk_qs(request.user, role_name))) - if or_filters: - q = Q() - for n, k, v in or_filters: - if n: - q |= ~Q(**{k: v}) - else: - q |= Q(**{k: v}) - args.append(q) - if search_filters and search_filter_relation == 'OR': - q = Q() - for term, constrains in search_filters.items(): - for constrain in constrains: - q |= Q(**{constrain: term}) - args.append(q) - elif search_filters and search_filter_relation == 'AND': - for term, constrains in search_filters.items(): - q_chain = Q() - for constrain in constrains: - q_chain |= Q(**{constrain: term}) - queryset = queryset.filter(q_chain) - for n, k, v in chain_filters: - if n: - q = ~Q(**{k: v}) - else: - q = Q(**{k: v}) - queryset = queryset.filter(q) - queryset = queryset.filter(*args) - if needs_distinct: - queryset = queryset.distinct() - return queryset - except (FieldError, FieldDoesNotExist, ValueError, TypeError) as e: - raise ParseError(e.args[0]) - except ValidationError as e: - raise ParseError(json.dumps(e.messages, ensure_ascii=False)) - - -class OrderByBackend(BaseFilterBackend): - """ - Filter to apply ordering based on query string parameters. - """ - - def filter_queryset(self, request, queryset, view): - try: - order_by = None - for key, value in request.query_params.items(): - if key in ('order', 'order_by'): - order_by = value - if ',' in value: - order_by = value.split(',') - else: - order_by = (value,) - default_order_by = self.get_default_ordering(view) - # glue the order by and default order by together so that the default is the backup option - order_by = list(order_by or []) + list(default_order_by or []) - if order_by: - order_by = self._validate_ordering_fields(queryset.model, order_by) - # Special handling of the type field for ordering. In this - # case, we're not sorting exactly on the type field, but - # given the limited number of views with multiple types, - # sorting on polymorphic_ctype.model is effectively the same. - new_order_by = [] - if 'polymorphic_ctype' in get_all_field_names(queryset.model): - for field in order_by: - if field == 'type': - new_order_by.append('polymorphic_ctype__model') - elif field == '-type': - new_order_by.append('-polymorphic_ctype__model') - else: - new_order_by.append(field) - else: - for field in order_by: - if field not in ('type', '-type'): - new_order_by.append(field) - queryset = queryset.order_by(*new_order_by) - return queryset - except FieldError as e: - # Return a 400 for invalid field names. - raise ParseError(*e.args) - - def get_default_ordering(self, view): - ordering = getattr(view, 'ordering', None) - if isinstance(ordering, str): - return (ordering,) - return ordering - - def _validate_ordering_fields(self, model, order_by): - for field_name in order_by: - # strip off the negation prefix `-` if it exists - prefix = '' - path = field_name - if field_name[0] == '-': - prefix = field_name[0] - path = field_name[1:] - try: - field, new_path = get_field_from_path(model, path) - new_path = '{}{}'.format(prefix, new_path) - except (FieldError, FieldDoesNotExist) as e: - raise ParseError(e.args[0]) - yield new_path diff --git a/awx/api/generics.py b/awx/api/generics.py index 0c16a3790f..1081b02c72 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -30,12 +30,13 @@ from rest_framework.permissions import IsAuthenticated from rest_framework.renderers import StaticHTMLRenderer from rest_framework.negotiation import DefaultContentNegotiation +from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend +from ansible_base.utils.models import get_all_field_names + # AWX -from awx.api.filters import FieldLookupBackend from awx.main.models import UnifiedJob, UnifiedJobTemplate, User, Role, Credential, WorkflowJobTemplateNode, WorkflowApprovalTemplate from awx.main.access import optimize_queryset from awx.main.utils import camelcase_to_underscore, get_search_fields, getattrd, get_object_or_400, decrypt_field, get_awx_version -from awx.main.utils.db import get_all_field_names from awx.main.utils.licensing import server_product_name from awx.main.views import ApiErrorView from awx.api.serializers import ResourceAccessListElementSerializer, CopySerializer diff --git a/awx/api/serializers.py b/awx/api/serializers.py index d0622bf23b..523b21af82 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -43,6 +43,8 @@ from rest_framework.utils.serializer_helpers import ReturnList # Django-Polymorphic from polymorphic.models import PolymorphicModel +from ansible_base.utils.models import get_type_for_model + # AWX from awx.main.access import get_user_capabilities from awx.main.constants import ACTIVE_STATES, CENSOR_VALUE @@ -102,7 +104,6 @@ from awx.main.models.base import VERBOSITY_CHOICES, NEW_JOB_TYPE_CHOICES from awx.main.models.rbac import role_summary_fields_generator, RoleAncestorEntry from awx.main.fields import ImplicitRoleField from awx.main.utils import ( - get_type_for_model, get_model_for_type, camelcase_to_underscore, getattrd, diff --git a/awx/conf/models.py b/awx/conf/models.py index 25cf0cd584..91274140fb 100644 --- a/awx/conf/models.py +++ b/awx/conf/models.py @@ -7,8 +7,10 @@ import json # Django from django.db import models +from ansible_base.utils.models import prevent_search + # AWX -from awx.main.models.base import CreatedModifiedModel, prevent_search +from awx.main.models.base import CreatedModifiedModel from awx.main.utils import encrypt_field from awx.conf import settings_registry diff --git a/awx/main/access.py b/awx/main/access.py index 9e40d1fb50..6b57723e5b 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -20,11 +20,12 @@ from rest_framework.exceptions import ParseError, PermissionDenied # Django OAuth Toolkit from awx.main.models.oauth import OAuth2Application, OAuth2AccessToken +from ansible_base.utils.validation import to_python_boolean + # AWX from awx.main.utils import ( get_object_or_400, get_pk_from_dict, - to_python_boolean, get_licenser, ) from awx.main.models import ( diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index 8803842fd0..53b7ea63cd 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -6,8 +6,10 @@ from django.conf import settings # noqa from django.db import connection from django.db.models.signals import pre_delete # noqa +from ansible_base.utils.models import prevent_search + # AWX -from awx.main.models.base import BaseModel, PrimordialModel, prevent_search, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa +from awx.main.models.base import BaseModel, PrimordialModel, accepts_json, CLOUD_INVENTORY_SOURCES, VERBOSITY_CHOICES # noqa from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutMaxBytesExceeded # noqa from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa diff --git a/awx/main/models/ad_hoc_commands.py b/awx/main/models/ad_hoc_commands.py index 3b71119031..9143f30f98 100644 --- a/awx/main/models/ad_hoc_commands.py +++ b/awx/main/models/ad_hoc_commands.py @@ -12,9 +12,11 @@ from django.utils.text import Truncator from django.utils.translation import gettext_lazy as _ from django.core.exceptions import ValidationError +from ansible_base.utils.models import prevent_search + # AWX from awx.api.versioning import reverse -from awx.main.models.base import prevent_search, AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty +from awx.main.models.base import AD_HOC_JOB_TYPE_CHOICES, VERBOSITY_CHOICES, VarsDictProperty from awx.main.models.events import AdHocCommandEvent, UnpartitionedAdHocCommandEvent from awx.main.models.unified_jobs import UnifiedJob from awx.main.models.notifications import JobNotificationMixin, NotificationTemplate diff --git a/awx/main/models/base.py b/awx/main/models/base.py index 0ef5b244f2..ce96d0bd31 100644 --- a/awx/main/models/base.py +++ b/awx/main/models/base.py @@ -15,7 +15,6 @@ from awx.main.utils import encrypt_field, parse_yaml_or_json from awx.main.constants import CLOUD_PROVIDERS __all__ = [ - 'prevent_search', 'VarsDictProperty', 'BaseModel', 'CreatedModifiedModel', @@ -384,23 +383,6 @@ class NotificationFieldsModel(BaseModel): notification_templates_started = models.ManyToManyField("NotificationTemplate", blank=True, related_name='%(class)s_notification_templates_for_started') -def prevent_search(relation): - """ - Used to mark a model field or relation as "restricted from filtering" - e.g., - - class AuthToken(BaseModel): - user = prevent_search(models.ForeignKey(...)) - sensitive_data = prevent_search(models.CharField(...)) - - The flag set by this function is used by - `awx.api.filters.FieldLookupBackend` to block fields and relations that - should not be searchable/filterable via search query params - """ - setattr(relation, '__prevent_search__', True) - return relation - - def accepts_json(relation): """ Used to mark a model field as allowing JSON e.g,. JobTemplate.extra_vars diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index 5d370d24c9..08490234f2 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -17,6 +17,8 @@ from django.db.models import Sum, Q import redis from solo.models import SingletonModel +from ansible_base.utils.models import prevent_search + # AWX from awx import __version__ as awx_application_version from awx.main.utils import is_testing @@ -24,7 +26,7 @@ from awx.api.versioning import reverse from awx.main.fields import ImplicitRoleField from awx.main.managers import InstanceManager, UUID_DEFAULT from awx.main.constants import JOB_FOLDER_PREFIX -from awx.main.models.base import BaseModel, HasEditsMixin, prevent_search +from awx.main.models.base import BaseModel, HasEditsMixin from awx.main.models.rbac import ( ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR, diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 7cf7f0710e..ab8ae3284b 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -25,6 +25,8 @@ from django.db.models import Q # REST Framework from rest_framework.exceptions import ParseError +from ansible_base.utils.models import prevent_search + # AWX from awx.api.versioning import reverse from awx.main.constants import CLOUD_PROVIDERS @@ -35,7 +37,7 @@ from awx.main.fields import ( OrderedManyToManyField, ) from awx.main.managers import HostManager, HostMetricActiveManager -from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, prevent_search, accepts_json +from awx.main.models.base import BaseModel, CommonModelNameNotUnique, VarsDictProperty, CLOUD_INVENTORY_SOURCES, accepts_json from awx.main.models.events import InventoryUpdateEvent, UnpartitionedInventoryUpdateEvent from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate from awx.main.models.mixins import ( diff --git a/awx/main/models/jobs.py b/awx/main/models/jobs.py index edbc3de0bd..a01f791af6 100644 --- a/awx/main/models/jobs.py +++ b/awx/main/models/jobs.py @@ -20,13 +20,14 @@ from django.core.exceptions import FieldDoesNotExist # REST Framework from rest_framework.exceptions import ParseError +from ansible_base.utils.models import prevent_search + # AWX from awx.api.versioning import reverse from awx.main.constants import HOST_FACTS_FIELDS from awx.main.models.base import ( BaseModel, CreatedModifiedModel, - prevent_search, accepts_json, JOB_TYPE_CHOICES, NEW_JOB_TYPE_CHOICES, diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 3c8d8ae1ef..fd92b0b5c3 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -16,8 +16,9 @@ from django.db.models.query import QuerySet from django.utils.crypto import get_random_string from django.utils.translation import gettext_lazy as _ +from ansible_base.utils.models import prevent_search + # AWX -from awx.main.models.base import prevent_search from awx.main.models.rbac import Role, RoleAncestorEntry from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic from awx.main.utils.execution_environments import get_default_execution_environment diff --git a/awx/main/models/notifications.py b/awx/main/models/notifications.py index ef8304b6a6..4b5ce89fca 100644 --- a/awx/main/models/notifications.py +++ b/awx/main/models/notifications.py @@ -15,9 +15,11 @@ from django.utils.encoding import smart_str, force_str from jinja2 import sandbox, ChainableUndefined from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError +from ansible_base.utils.models import prevent_search + # AWX from awx.api.versioning import reverse -from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel, prevent_search +from awx.main.models.base import CommonModelNameNotUnique, CreatedModifiedModel from awx.main.utils import encrypt_field, decrypt_field, set_environ from awx.main.notifications.email_backend import CustomEmailBackend from awx.main.notifications.slack_backend import SlackBackend diff --git a/awx/main/models/unified_jobs.py b/awx/main/models/unified_jobs.py index 6ba605c0d4..83696d1eef 100644 --- a/awx/main/models/unified_jobs.py +++ b/awx/main/models/unified_jobs.py @@ -30,8 +30,10 @@ from rest_framework.exceptions import ParseError # Django-Polymorphic from polymorphic.models import PolymorphicModel +from ansible_base.utils.models import prevent_search, get_type_for_model + # AWX -from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel, prevent_search +from awx.main.models.base import CommonModelNameNotUnique, PasswordFieldsModel, NotificationFieldsModel from awx.main.dispatch import get_task_queuename from awx.main.dispatch.control import Control as ControlDispatcher from awx.main.registrar import activity_stream_registrar @@ -42,7 +44,6 @@ from awx.main.utils.common import ( _inventory_updates, copy_model_by_class, copy_m2m_relationships, - get_type_for_model, parse_yaml_or_json, getattr_dne, ScheduleDependencyManager, diff --git a/awx/main/models/workflow.py b/awx/main/models/workflow.py index 3038db2967..5f0cbff7ca 100644 --- a/awx/main/models/workflow.py +++ b/awx/main/models/workflow.py @@ -23,9 +23,11 @@ from crum import get_current_user from jinja2 import sandbox from jinja2.exceptions import TemplateSyntaxError, UndefinedError, SecurityError +from ansible_base.utils.models import prevent_search + # AWX from awx.api.versioning import reverse -from awx.main.models import prevent_search, accepts_json, UnifiedJobTemplate, UnifiedJob +from awx.main.models import accepts_json, UnifiedJobTemplate, UnifiedJob from awx.main.models.notifications import NotificationTemplate, JobNotificationMixin from awx.main.models.base import CreatedModifiedModel, VarsDictProperty from awx.main.models.rbac import ROLE_SINGLETON_SYSTEM_ADMINISTRATOR, ROLE_SINGLETON_SYSTEM_AUDITOR diff --git a/awx/main/scheduler/task_manager.py b/awx/main/scheduler/task_manager.py index 441c4e921b..18ebec4275 100644 --- a/awx/main/scheduler/task_manager.py +++ b/awx/main/scheduler/task_manager.py @@ -17,6 +17,8 @@ from django.utils.timezone import now as tz_now from django.conf import settings from django.contrib.contenttypes.models import ContentType +from ansible_base.utils.models import get_type_for_model + # AWX from awx.main.dispatch.reaper import reap_job from awx.main.models import ( @@ -34,7 +36,6 @@ from awx.main.models import ( from awx.main.scheduler.dag_workflow import WorkflowDAG from awx.main.utils.pglock import advisory_lock from awx.main.utils import ( - get_type_for_model, ScheduleTaskManager, ScheduleWorkflowManager, ) diff --git a/awx/main/tests/functional/api/test_survey_spec.py b/awx/main/tests/functional/api/test_survey_spec.py index cbb22b3bdc..97a7f65c0f 100644 --- a/awx/main/tests/functional/api/test_survey_spec.py +++ b/awx/main/tests/functional/api/test_survey_spec.py @@ -2,12 +2,12 @@ from unittest import mock import pytest import json +from ansible_base.utils.models import get_type_for_model from awx.api.versioning import reverse from awx.main.models.jobs import JobTemplate, Job from awx.main.models.activity_stream import ActivityStream from awx.main.access import JobTemplateAccess -from awx.main.utils.common import get_type_for_model @pytest.fixture diff --git a/awx/main/tests/unit/api/test_filters.py b/awx/main/tests/unit/api/test_filters.py index 7d6501a871..0066fdd0f6 100644 --- a/awx/main/tests/unit/api/test_filters.py +++ b/awx/main/tests/unit/api/test_filters.py @@ -3,15 +3,13 @@ import pytest # Django -from django.core.exceptions import FieldDoesNotExist +from rest_framework.exceptions import PermissionDenied -from rest_framework.exceptions import PermissionDenied, ParseError +from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend -from awx.api.filters import FieldLookupBackend, OrderByBackend, get_field_from_path from awx.main.models import ( AdHocCommand, ActivityStream, - Credential, Job, JobTemplate, SystemJob, @@ -20,88 +18,11 @@ from awx.main.models import ( WorkflowJob, WorkflowJobTemplate, WorkflowJobOptions, - InventorySource, - JobEvent, ) from awx.main.models.oauth import OAuth2Application from awx.main.models.jobs import JobOptions -def test_related(): - field_lookup = FieldLookupBackend() - lookup = '__'.join(['inventory', 'organization', 'pk']) - field, new_lookup = field_lookup.get_field_from_lookup(InventorySource, lookup) - print(field) - print(new_lookup) - - -def test_invalid_filter_key(): - field_lookup = FieldLookupBackend() - # FieldDoesNotExist is caught and converted to ParseError by filter_queryset - with pytest.raises(FieldDoesNotExist) as excinfo: - field_lookup.value_to_python(JobEvent, 'event_data.task_action', 'foo') - assert 'has no field named' in str(excinfo) - - -def test_invalid_field_hop(): - with pytest.raises(ParseError) as excinfo: - get_field_from_path(Credential, 'organization__description__user') - assert 'No related model for' in str(excinfo) - - -def test_invalid_order_by_key(): - field_order_by = OrderByBackend() - with pytest.raises(ParseError) as excinfo: - [f for f in field_order_by._validate_ordering_fields(JobEvent, ('event_data.task_action',))] - assert 'has no field named' in str(excinfo) - - -@pytest.mark.parametrize(u"empty_value", [u'', '']) -def test_empty_in(empty_value): - field_lookup = FieldLookupBackend() - with pytest.raises(ValueError) as excinfo: - field_lookup.value_to_python(JobTemplate, 'project__name__in', empty_value) - assert 'empty value for __in' in str(excinfo.value) - - -@pytest.mark.parametrize(u"valid_value", [u'foo', u'foo,']) -def test_valid_in(valid_value): - field_lookup = FieldLookupBackend() - value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__in', valid_value) - assert 'foo' in value - - -def test_invalid_field(): - invalid_field = u"ヽヾ" - field_lookup = FieldLookupBackend() - with pytest.raises(ValueError) as excinfo: - field_lookup.value_to_python(WorkflowJobTemplate, invalid_field, 'foo') - assert 'is not an allowed field name. Must be ascii encodable.' in str(excinfo.value) - - -def test_valid_iexact(): - field_lookup = FieldLookupBackend() - value, new_lookup, _ = field_lookup.value_to_python(JobTemplate, 'project__name__iexact', 'foo') - assert 'foo' in value - - -def test_invalid_iexact(): - field_lookup = FieldLookupBackend() - with pytest.raises(ValueError) as excinfo: - field_lookup.value_to_python(Job, 'id__iexact', '1') - assert 'is not a text field and cannot be filtered by case-insensitive search' in str(excinfo.value) - - -@pytest.mark.parametrize('lookup_suffix', ['', 'contains', 'startswith', 'in']) -@pytest.mark.parametrize('password_field', Credential.PASSWORD_FIELDS) -def test_filter_on_password_field(password_field, lookup_suffix): - field_lookup = FieldLookupBackend() - lookup = '__'.join(filter(None, [password_field, lookup_suffix])) - with pytest.raises(PermissionDenied) as excinfo: - field, new_lookup = field_lookup.get_field_from_lookup(Credential, lookup) - assert 'not allowed' in str(excinfo.value) - - @pytest.mark.parametrize( 'model, query', [ @@ -128,10 +49,3 @@ def test_filter_sensitive_fields_and_relations(model, query): with pytest.raises(PermissionDenied) as excinfo: field, new_lookup = field_lookup.get_field_from_lookup(model, query) assert 'not allowed' in str(excinfo.value) - - -def test_looping_filters_prohibited(): - field_lookup = FieldLookupBackend() - with pytest.raises(ParseError) as loop_exc: - field_lookup.get_field_from_lookup(Job, 'job_events__job__job_events') - assert 'job_events' in str(loop_exc.value) diff --git a/awx/main/tests/unit/utils/test_common.py b/awx/main/tests/unit/utils/test_common.py index cc8f65bf93..1d960c45db 100644 --- a/awx/main/tests/unit/utils/test_common.py +++ b/awx/main/tests/unit/utils/test_common.py @@ -12,6 +12,8 @@ from unittest import mock from rest_framework.exceptions import ParseError +from ansible_base.utils.models import get_type_for_model + from awx.main.utils import common from awx.api.validators import HostnameRegexValidator @@ -106,7 +108,7 @@ TEST_MODELS = [ # Cases relied on for scheduler dependent jobs list @pytest.mark.parametrize('model,name', TEST_MODELS) def test_get_type_for_model(model, name): - assert common.get_type_for_model(model) == name + assert get_type_for_model(model) == name def test_get_model_for_invalid_type(): diff --git a/awx/main/tests/unit/utils/test_filters.py b/awx/main/tests/unit/utils/test_filters.py index cf07b87d65..ad1115043f 100644 --- a/awx/main/tests/unit/utils/test_filters.py +++ b/awx/main/tests/unit/utils/test_filters.py @@ -68,7 +68,9 @@ class mockHost: @mock.patch('awx.main.utils.filters.get_model', return_value=mockHost()) class TestSmartFilterQueryFromString: - @mock.patch('awx.api.filters.get_fields_from_path', lambda model, path: ([model], path)) # disable field filtering, because a__b isn't a real Host field + @mock.patch( + 'ansible_base.filters.rest_framework.field_lookup_backend.get_fields_from_path', lambda model, path: ([model], path) + ) # disable field filtering, because a__b isn't a real Host field @pytest.mark.parametrize( "filter_string,q_expected", [ diff --git a/awx/main/utils/common.py b/awx/main/utils/common.py index 9066707d4d..7ab9842584 100644 --- a/awx/main/utils/common.py +++ b/awx/main/utils/common.py @@ -52,12 +52,10 @@ __all__ = [ 'get_awx_http_client_headers', 'get_awx_version', 'update_scm_url', - 'get_type_for_model', 'get_model_for_type', 'copy_model_by_class', 'copy_m2m_relationships', 'prefetch_page_capabilities', - 'to_python_boolean', 'datetime_hook', 'ignore_inventory_computed_fields', 'ignore_inventory_group_removal', @@ -110,18 +108,6 @@ def get_object_or_400(klass, *args, **kwargs): raise ParseError(*e.args) -def to_python_boolean(value, allow_none=False): - value = str(value) - if value.lower() in ('true', '1', 't'): - return True - elif value.lower() in ('false', '0', 'f'): - return False - elif allow_none and value.lower() in ('none', 'null'): - return None - else: - raise ValueError(_(u'Unable to convert "%s" to boolean') % value) - - def datetime_hook(d): new_d = {} for key, value in d.items(): @@ -569,14 +555,6 @@ def copy_m2m_relationships(obj1, obj2, fields, kwargs=None): dest_field.add(*list(src_field_value.all().values_list('id', flat=True))) -def get_type_for_model(model): - """ - Return type name for a given model class. - """ - opts = model._meta.concrete_model._meta - return camelcase_to_underscore(opts.object_name) - - def get_model_for_type(type_name): """ Return model class for a given type name. diff --git a/awx/main/utils/db.py b/awx/main/utils/db.py index 4117c5274c..8cc6aacce9 100644 --- a/awx/main/utils/db.py +++ b/awx/main/utils/db.py @@ -1,27 +1,10 @@ # Copyright (c) 2017 Ansible by Red Hat # All Rights Reserved. -from itertools import chain from awx.settings.application_name import set_application_name from django.conf import settings -def get_all_field_names(model): - # Implements compatibility with _meta.get_all_field_names - # See: https://docs.djangoproject.com/en/1.11/ref/models/meta/#migrating-from-the-old-api - return list( - set( - chain.from_iterable( - (field.name, field.attname) if hasattr(field, 'attname') else (field.name,) - for field in model._meta.get_fields() - # For complete backwards compatibility, you may want to exclude - # GenericForeignKey from the results. - if not (field.many_to_one and field.related_model is None) - ) - ) - ) - - def set_connection_name(function): set_application_name(settings.DATABASES, settings.CLUSTER_HOST_ID, function=function) diff --git a/awx/main/utils/filters.py b/awx/main/utils/filters.py index 7f9724329b..682938cb16 100644 --- a/awx/main/utils/filters.py +++ b/awx/main/utils/filters.py @@ -161,7 +161,7 @@ class SmartFilter(object): else: # detect loops and restrict access to sensitive fields # this import is intentional here to avoid a circular import - from awx.api.filters import FieldLookupBackend + from ansible_base.filters.rest_framework.field_lookup_backend import FieldLookupBackend FieldLookupBackend().get_field_from_lookup(Host, k) kwargs[k] = v diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 8bdb9617ac..85bc45f7eb 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -11,6 +11,7 @@ from datetime import timedelta # python-ldap import ldap +from split_settings.tools import include DEBUG = True @@ -354,12 +355,6 @@ REST_FRAMEWORK = { 'awx.api.authentication.LoggedBasicAuthentication', ), 'DEFAULT_PERMISSION_CLASSES': ('awx.api.permissions.ModelAccessPermission',), - 'DEFAULT_FILTER_BACKENDS': ( - 'awx.api.filters.TypeFilterBackend', - 'awx.api.filters.FieldLookupBackend', - 'rest_framework.filters.SearchFilter', - 'awx.api.filters.OrderByBackend', - ), 'DEFAULT_PARSER_CLASSES': ('awx.api.parsers.JSONParser',), 'DEFAULT_RENDERER_CLASSES': ('awx.api.renderers.DefaultJSONRenderer', 'awx.api.renderers.BrowsableAPIRenderer'), 'DEFAULT_METADATA_CLASS': 'awx.api.metadata.Metadata', @@ -1067,3 +1062,12 @@ CLEANUP_HOST_METRICS_HARD_THRESHOLD = 36 # months # Host metric summary monthly task - last time of run HOST_METRIC_SUMMARY_TASK_LAST_TS = None HOST_METRIC_SUMMARY_TASK_INTERVAL = 7 # days + + +# django-ansible-base +ANSIBLE_BASE_FEATURES = {'AUTHENTICATION': False, 'SWAGGER': False, 'FILTERING': True} + +from ansible_base import settings # noqa: E402 + +settings_file = os.path.join(os.path.dirname(settings.__file__), 'dynamic_settings.py') +include(settings_file)