diff --git a/awx/api/filters.py b/awx/api/filters.py index 050534a963..af0060c008 100644 --- a/awx/api/filters.py +++ b/awx/api/filters.py @@ -19,6 +19,11 @@ from rest_framework.filters import BaseFilterBackend # Ansible Tower from awx.main.utils import get_type_for_model, to_python_boolean +class MongoFilterBackend(BaseFilterBackend): + + def filter_queryset(self, request, queryset, view): + return queryset + class ActiveOnlyBackend(BaseFilterBackend): ''' Filter to show only objects where is_active/active is True. @@ -61,7 +66,7 @@ class TypeFilterBackend(BaseFilterBackend): queryset = queryset.filter(polymorphic_ctype_id__in=types_pks) elif model_type in types: queryset = queryset - else: + else: queryset = queryset.none() return queryset except FieldError, e: diff --git a/awx/api/generics.py b/awx/api/generics.py index 1ac01a3eb6..b5e90ee876 100644 --- a/awx/api/generics.py +++ b/awx/api/generics.py @@ -31,7 +31,8 @@ __all__ = ['APIView', 'GenericAPIView', 'ListAPIView', 'SimpleListAPIView', 'ListCreateAPIView', 'SubListAPIView', 'SubListCreateAPIView', 'SubListCreateAttachDetachAPIView', 'RetrieveAPIView', 'RetrieveUpdateAPIView', 'RetrieveDestroyAPIView', - 'RetrieveUpdateDestroyAPIView', 'DestroyAPIView'] + 'RetrieveUpdateDestroyAPIView', 'DestroyAPIView', + 'MongoAPIView', 'MongoListAPIView'] logger = logging.getLogger('awx.api.generics') @@ -164,7 +165,6 @@ class APIView(views.APIView): ret['added_in_version'] = added_in_version return ret - class GenericAPIView(generics.GenericAPIView, APIView): # Base class for all model-based views. @@ -195,11 +195,13 @@ class GenericAPIView(generics.GenericAPIView, APIView): if not hasattr(self, 'format_kwarg'): self.format_kwarg = 'format' d = super(GenericAPIView, self).get_description_context() - d.update({ - 'model_verbose_name': unicode(self.model._meta.verbose_name), - 'model_verbose_name_plural': unicode(self.model._meta.verbose_name_plural), - 'serializer_fields': self.get_serializer().metadata(), - }) + if hasattr(self.model, "_meta"): + if hasattr(self.model._meta, "verbose_name"): + d.update({ + 'model_verbose_name': unicode(self.model._meta.verbose_name), + 'model_verbose_name_plural': unicode(self.model._meta.verbose_name_plural), + }) + d.update({'serializer_fields': self.get_serializer().metadata()}) return d def metadata(self, request): @@ -252,6 +254,27 @@ class GenericAPIView(generics.GenericAPIView, APIView): ret['search_fields'] = self.search_fields return ret +class MongoAPIView(GenericAPIView): + + def get_parent_object(self): + parent_filter = { + self.lookup_field: self.kwargs.get(self.lookup_field, None), + } + return get_object_or_404(self.parent_model, **parent_filter) + + def check_parent_access(self, parent=None): + parent = parent or self.get_parent_object() + parent_access = getattr(self, 'parent_access', 'read') + if parent_access in ('read', 'delete'): + args = (self.parent_model, parent_access, parent) + else: + args = (self.parent_model, parent_access, parent, None) + if not self.request.user.can_access(*args): + raise PermissionDenied() + +class MongoListAPIView(generics.ListAPIView, MongoAPIView): + pass + class SimpleListAPIView(generics.ListAPIView, GenericAPIView): def get_queryset(self): diff --git a/awx/api/serializers.py b/awx/api/serializers.py index 65b573de11..ee6519ffb9 100644 --- a/awx/api/serializers.py +++ b/awx/api/serializers.py @@ -8,6 +8,9 @@ import logging from dateutil import rrule from ast import literal_eval +import mongoengine +from rest_framework_mongoengine.serializers import MongoEngineModelSerializer + # PyYAML import yaml @@ -36,6 +39,8 @@ from awx.main.models import * # noqa from awx.main.utils import get_type_for_model, get_model_for_type from awx.main.redact import REPLACE_STR +from awx.fact.models import * # noqa + logger = logging.getLogger('awx.api.serializers') # Fields that should be summarized regardless of object type. @@ -1537,12 +1542,10 @@ class JobRelaunchSerializer(JobSerializer): obj = self.context.get('obj') if not obj.credential or obj.credential.active is False: raise serializers.ValidationError(dict(credential=["Credential not found or deleted."])) - if obj.job_type != PERM_INVENTORY_SCAN and (obj.project is None or not obj.project.active): raise serializers.ValidationError(dict(errors=["Job Template Project is missing or undefined"])) if obj.inventory is None or not obj.inventory.active: raise serializers.ValidationError(dict(errors=["Job Template Inventory is missing or undefined"])) - return attrs class AdHocCommandSerializer(UnifiedJobSerializer): @@ -2010,3 +2013,17 @@ class AuthTokenSerializer(serializers.Serializer): raise serializers.ValidationError('Unable to login with provided credentials.') else: raise serializers.ValidationError('Must include "username" and "password"') + + +class FactVersionSerializer(MongoEngineModelSerializer): + + class Meta: + model = FactVersion + fields = ('module', 'timestamp',) + +class FactSerializer(MongoEngineModelSerializer): + + class Meta: + model = Fact + depth = 2 + fields = ('timestamp', 'host', 'module', 'fact') diff --git a/awx/api/urls.py b/awx/api/urls.py index 16d5ddb2ea..0351be1ac8 100644 --- a/awx/api/urls.py +++ b/awx/api/urls.py @@ -75,6 +75,7 @@ inventory_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/activity_stream/$', 'inventory_activity_stream_list'), url(r'^(?P[0-9]+)/scan_job_templates/$', 'inventory_scan_job_template_list'), url(r'^(?P[0-9]+)/ad_hoc_commands/$', 'inventory_ad_hoc_commands_list'), + url(r'^(?P[0-9]+)/single_fact/$', 'inventory_single_fact_view'), ) host_urls = patterns('awx.api.views', @@ -89,6 +90,9 @@ host_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/inventory_sources/$', 'host_inventory_sources_list'), url(r'^(?P[0-9]+)/ad_hoc_commands/$', 'host_ad_hoc_commands_list'), url(r'^(?P[0-9]+)/ad_hoc_command_events/$', 'host_ad_hoc_command_events_list'), + url(r'^(?P[0-9]+)/single_fact/$', 'host_single_fact_view'), + url(r'^(?P[0-9]+)/fact_versions/$', 'host_fact_versions_list'), + url(r'^(?P[0-9]+)/fact_view/$', 'host_fact_compare_view'), ) group_urls = patterns('awx.api.views', @@ -104,6 +108,7 @@ group_urls = patterns('awx.api.views', url(r'^(?P[0-9]+)/activity_stream/$', 'group_activity_stream_list'), url(r'^(?P[0-9]+)/inventory_sources/$', 'group_inventory_sources_list'), url(r'^(?P[0-9]+)/ad_hoc_commands/$', 'group_ad_hoc_commands_list'), + url(r'^(?P[0-9]+)/single_fact/$', 'group_single_fact_view'), ) inventory_source_urls = patterns('awx.api.views', diff --git a/awx/api/views.py b/awx/api/views.py index e52dd77009..e81f8845d0 100644 --- a/awx/api/views.py +++ b/awx/api/views.py @@ -46,6 +46,7 @@ from awx.main.access import get_user_queryset from awx.main.ha import is_ha_environment from awx.api.authentication import TaskAuthentication from awx.api.utils.decorators import paginated +from awx.api.filters import MongoFilterBackend from awx.api.generics import get_view_name from awx.api.generics import * # noqa from awx.main.models import * # noqa @@ -53,6 +54,7 @@ from awx.main.utils import * # noqa from awx.api.permissions import * # noqa from awx.api.renderers import * # noqa from awx.api.serializers import * # noqa +from awx.fact.models import * # noqa def api_exception_handler(exc): ''' @@ -922,6 +924,27 @@ class InventoryScanJobTemplateList(SubListAPIView): qs = self.request.user.get_queryset(self.model) return qs.filter(job_type=PERM_INVENTORY_SCAN, inventory=parent) +class InventorySingleFactView(MongoAPIView): + + model = Fact + parent_model = Inventory + new_in_220 = True + serializer_class = FactSerializer + filter_backends = (MongoFilterBackend,) + + def get(self, request, *args, **kwargs): + fact_key = request.QUERY_PARAMS.get("fact_key", None) + fact_value = request.QUERY_PARAMS.get("fact_value", None) + datetime_spec = request.QUERY_PARAMS.get("timestamp", None) + module_spec = request.QUERY_PARAMS.get("module", None) + + if fact_key is None or fact_value is None or module_spec is None: + return Response({"error": "Missing fields"}, status=status.HTTP_400_BAD_REQUEST) + datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() + inventory_obj = self.get_parent_object() + fact_data = Fact.get_single_facts([h.name for h in inventory_obj.hosts.all()], fact_key, fact_value, datetime_actual, module_spec) + return Response(FactSerializer(fact_data).data if fact_data is not None else {}) + class HostList(ListCreateAPIView): @@ -986,6 +1009,79 @@ class HostActivityStreamList(SubListAPIView): qs = self.request.user.get_queryset(self.model) return qs.filter(Q(host=parent) | Q(inventory=parent.inventory)) +class HostFactVersionsList(MongoListAPIView): + + serializer_class = FactVersionSerializer + parent_model = Host + new_in_220 = True + filter_backends = (MongoFilterBackend,) + + def get_queryset(self): + from_spec = self.request.QUERY_PARAMS.get('from', None) + to_spec = self.request.QUERY_PARAMS.get('to', None) + module_spec = self.request.QUERY_PARAMS.get('module', None) + + host = self.get_parent_object() + self.check_parent_access(host) + + try: + fact_host = FactHost.objects.get(hostname=host.name) + except FactHost.DoesNotExist: + return None + + kv = { + 'host': fact_host.id, + } + if module_spec is not None: + kv['module'] = module_spec + if from_spec is not None: + from_actual = dateutil.parser.parse(from_spec) + kv['timestamp__gt'] = from_actual + if from_spec is not None and to_spec is not None: + to_actual = dateutil.parser.parse(to_spec) + kv['timestamp__lte'] = to_actual + + return FactVersion.objects.filter(**kv).order_by("-timestamp") + +class HostSingleFactView(MongoAPIView): + + model = Fact + parent_model = Host + new_in_220 = True + serializer_class = FactSerializer + filter_backends = (MongoFilterBackend,) + + def get(self, request, *args, **kwargs): + fact_key = request.QUERY_PARAMS.get("fact_key", None) + fact_value = request.QUERY_PARAMS.get("fact_value", None) + datetime_spec = request.QUERY_PARAMS.get("timestamp", None) + module_spec = request.QUERY_PARAMS.get("module", None) + + if fact_key is None or fact_value is None or module_spec is None: + return Response({"error": "Missing fields"}, status=status.HTTP_400_BAD_REQUEST) + datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() + host_obj = self.get_parent_object() + fact_data = Fact.get_single_facts([host_obj.name], fact_key, fact_value, datetime_actual, module_spec) + return Response(FactSerializer(fact_data).data if fact_data is not None else {}) + +class HostFactCompareView(MongoAPIView): + + new_in_220 = True + parent_model = Host + serializer_class = FactSerializer + filter_backends = (MongoFilterBackend,) + + def get(self, request, *args, **kwargs): + datetime_spec = request.QUERY_PARAMS.get('datetime', None) + module_spec = request.QUERY_PARAMS.get('module', "ansible") + datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() + + host_obj = self.get_parent_object() + fact_entry = Fact.get_host_version(host_obj.name, datetime_actual, module_spec) + host_data = FactSerializer(fact_entry).data if fact_entry is not None else {} + + return Response(host_data) + class GroupList(ListCreateAPIView): @@ -1125,6 +1221,28 @@ class GroupDetail(RetrieveUpdateDestroyAPIView): obj.mark_inactive_recursive() return Response(status=status.HTTP_204_NO_CONTENT) + +class GroupSingleFactView(MongoAPIView): + + model = Fact + parent_model = Group + new_in_220 = True + serializer_class = FactSerializer + filter_backends = (MongoFilterBackend,) + + def get(self, request, *args, **kwargs): + fact_key = request.QUERY_PARAMS.get("fact_key", None) + fact_value = request.QUERY_PARAMS.get("fact_value", None) + datetime_spec = request.QUERY_PARAMS.get("timestamp", None) + module_spec = request.QUERY_PARAMS.get("module", None) + + if fact_key is None or fact_value is None or module_spec is None: + return Response({"error": "Missing fields"}, status=status.HTTP_400_BAD_REQUEST) + datetime_actual = dateutil.parser.parse(datetime_spec) if datetime_spec is not None else now() + group_obj = self.get_parent_object() + fact_data = Fact.get_single_facts([h.name for h in group_obj.hosts.all()], fact_key, fact_value, datetime_actual, module_spec) + return Response(FactSerializer(fact_data).data if fact_data is not None else {}) + class InventoryGroupsList(SubListCreateAttachDetachAPIView): model = Group diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py index 4a1c5e3313..f9d5796ca2 100644 --- a/awx/fact/__init__.py +++ b/awx/fact/__init__.py @@ -17,4 +17,4 @@ try: connect(settings.MONGO_DB) register_key_transform(get_db()) except ConnectionError: - logger.warn('Failed to establish connect to MongDB "%s"' % (settings.MONGO_DB)) + logger.warn('Failed to establish connect to MongoDB "%s"' % (settings.MONGO_DB)) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 34e44256ce..1a5d0f64ed 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -78,7 +78,7 @@ class Fact(Document): } try: - facts = Fact.objects.filter(**kv) + facts = Fact.objects.filter(**kv).order_by("-timestamp") if not facts: return None return facts[0] @@ -97,7 +97,7 @@ class Fact(Document): 'module': module, } - return FactVersion.objects.filter(**kv).values_list('timestamp') + return FactVersion.objects.filter(**kv).order_by("-timestamp").values_list('timestamp') @staticmethod def get_single_facts(hostnames, fact_key, fact_value, timestamp, module): @@ -126,6 +126,9 @@ class Fact(Document): } fields = { 'fact.%s.$' % fact_key : 1, + 'host': 1, + 'timestamp': 1, + 'module': 1, } facts = Fact._get_collection().find(kv, fields) #fact_objs = [Fact(**f) for f in facts] @@ -136,11 +139,10 @@ class Fact(Document): fact_objs.append(Fact(**f)) return fact_objs - class FactVersion(Document): timestamp = DateTimeField(required=True) host = ReferenceField(FactHost, required=True) - module = StringField(max_length=50, required=True) + module = StringField(max_length=50, required=True) fact = ReferenceField(Fact, required=True) # TODO: Consider using hashed index on module. django-mongo may not support this but # executing raw js will @@ -150,4 +152,3 @@ class FactVersion(Document): 'module' ] } - \ No newline at end of file