diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py deleted file mode 100644 index e484e62be1..0000000000 --- a/awx/fact/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved. diff --git a/awx/fact/management/__init__.py b/awx/fact/management/__init__.py deleted file mode 100644 index 3a75c16036..0000000000 --- a/awx/fact/management/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved diff --git a/awx/fact/models/__init__.py b/awx/fact/models/__init__.py deleted file mode 100644 index 049720a11a..0000000000 --- a/awx/fact/models/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved - -from __future__ import absolute_import - -from .fact import * # noqa diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py deleted file mode 100644 index f52abe1106..0000000000 --- a/awx/fact/models/fact.py +++ /dev/null @@ -1,217 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved - -from mongoengine import connect -from mongoengine.base import BaseField -from mongoengine import Document, DateTimeField, ReferenceField, StringField, IntField -from mongoengine.connection import get_db, ConnectionError -from awx.fact.utils.dbtransform import register_key_transform, KeyTransform - -from django.conf import settings - -import logging -logger = logging.getLogger('awx.fact.models.fact') - - -key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) - -# NOTE: I think it might be better to use register_connection here: https://github.com/MongoEngine/mongoengine/blob/0.9/mongoengine/connection.py#L21 -# but I'm not doing that because I don't see how we can also register the key transform as needed or set the tz_aware preference -@classmethod -def _get_db_monkeypatched(cls): - """ Override the default _get_db mechanism to start a connection to the database """ - # Connect to Mongo - try: - # Sanity check: If we have intentionally invalid settings, then we - # know we cannot connect. - if settings.MONGO_HOST == NotImplemented: - raise ConnectionError - - # Attempt to connect to the MongoDB database. - connect(settings.MONGO_DB, - host=settings.MONGO_HOST, - port=int(settings.MONGO_PORT), - username=settings.MONGO_USERNAME, - password=settings.MONGO_PASSWORD, - tz_aware=settings.USE_TZ) - register_key_transform(get_db()) - except (ConnectionError, AttributeError): - logger.info('Failed to establish connect to MongoDB') - return get_db(cls._meta.get("db_alias", "default")) - -Document._get_db = _get_db_monkeypatched - -class TransformField(BaseField): - def to_python(self, value): - return key_transform.transform_outgoing(value, None) - - def prepare_query_value(self, op, value): - if op == 'set': - value = key_transform.transform_incoming(value, None) - return super(TransformField, self).prepare_query_value(op, value) - - def to_mongo(self, value): - value = key_transform.transform_incoming(value, None) - return value - -class FactHost(Document): - hostname = StringField(max_length=100, required=True, unique_with='inventory_id') - inventory_id = IntField(required=True, unique_with='hostname') - - # TODO: Consider using hashed index on hostname. django-mongo may not support this but - # executing raw js will - meta = { - 'indexes': [ - ('hostname', 'inventory_id') - ] - } - -class Fact(Document): - timestamp = DateTimeField(required=True) - host = ReferenceField(FactHost, required=True) - module = StringField(max_length=50, required=True) - fact = TransformField(required=True) - - # TODO: Consider using hashed index on host. django-mongo may not support this but - # executing raw js will - meta = { - 'indexes': [ - '-timestamp', - 'host', - ] - } - - @staticmethod - def add_fact(timestamp, fact, host, module): - fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact) - fact_obj.save() - version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj) - version_obj.save() - return (fact_obj, version_obj) - - # TODO: if we want to relax the need to include module... - # If module not specified then filter query may return more than 1 result. - # Thus, the resulting facts must somehow be unioned/concated/ or kept as an array. - @staticmethod - def get_host_version(hostname, inventory_id, timestamp, module): - try: - host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id) - except FactHost.DoesNotExist: - return None - - kv = { - 'host' : host.id, - 'timestamp__lte': timestamp, - 'module': module, - } - - try: - facts = Fact.objects.filter(**kv).order_by("-timestamp") - if not facts: - return None - return facts[0] - except Fact.DoesNotExist: - return None - - @staticmethod - def get_host_timeline(hostname, inventory_id, module): - try: - host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id) - except FactHost.DoesNotExist: - return None - - kv = { - 'host': host.id, - 'module': module, - } - - return FactVersion.objects.filter(**kv).order_by("-timestamp").values_list('timestamp') - - # FIXME: single facts no longer works with the addition of the inventory_id field to the FactHost document - @staticmethod - def get_single_facts(hostnames, fact_key, fact_value, timestamp, module): - kv = { - 'hostname': { - '$in': hostnames, - } - } - fields = { - '_id': 1 - } - host_ids = FactHost._get_collection().find(kv, fields) - if not host_ids or host_ids.count() == 0: - return None - # TODO: use mongo to transform [{_id: <>}, {_id: <>},...] into [_id, _id,...] - host_ids = [e['_id'] for e in host_ids] - - pipeline = [] - match = { - 'host': { - '$in': host_ids - }, - 'timestamp': { - '$lte': timestamp - }, - 'module': module - } - sort = { - 'timestamp': -1 - } - group = { - '_id': '$host', - 'timestamp': { - '$first': '$timestamp' - }, - 'fact': { - '$first': '$fact' - } - } - project = { - '_id': 0, - 'fact': 1, - } - pipeline.append({'$match': match}) # noqa - pipeline.append({'$sort': sort}) # noqa - pipeline.append({'$group': group}) # noqa - pipeline.append({'$project': project}) # noqa - q = FactVersion._get_collection().aggregate(pipeline) - if not q or 'result' not in q or len(q['result']) == 0: - return None - # TODO: use mongo to transform [{fact: <>}, {fact: <>},...] into [fact, fact,...] - fact_ids = [fact['fact'] for fact in q['result']] - - kv = { - 'fact.%s' % fact_key : fact_value, - '_id': { - '$in': fact_ids - } - } - fields = { - 'fact.%s.$' % fact_key : 1, - 'host': 1, - 'timestamp': 1, - 'module': 1, - } - facts = Fact._get_collection().find(kv, fields) - #fact_objs = [Fact(**f) for f in facts] - # Translate pymongo python structure to mongoengine Fact object - fact_objs = [] - for f in facts: - f['id'] = f.pop('_id') - fact_objs.append(Fact(**f)) - return fact_objs - -class FactVersion(Document): - timestamp = DateTimeField(required=True) - host = ReferenceField(FactHost, required=True) - module = StringField(max_length=50, required=True) - fact = ReferenceField(Fact, required=True) - # TODO: Consider using hashed index on module. django-mongo may not support this but - # executing raw js will - meta = { - 'indexes': [ - '-timestamp', - 'module', - 'host', - ] - } diff --git a/awx/fact/utils/__init__.py b/awx/fact/utils/__init__.py deleted file mode 100644 index 3a75c16036..0000000000 --- a/awx/fact/utils/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved diff --git a/awx/fact/utils/dbtransform.py b/awx/fact/utils/dbtransform.py deleted file mode 100644 index 0541c32fe0..0000000000 --- a/awx/fact/utils/dbtransform.py +++ /dev/null @@ -1,58 +0,0 @@ -# Copyright (c) 2015 Ansible, Inc. -# All Rights Reserved. - -# Pymongo -from pymongo.son_manipulator import SONManipulator - -class KeyTransform(SONManipulator): - - def __init__(self, replace): - self.replace = replace - - def replace_key(self, key): - for (replace, replacement) in self.replace: - key = key.replace(replace, replacement) - return key - - def revert_key(self, key): - for (replacement, replace) in self.replace: - key = key.replace(replace, replacement) - return key - - def replace_incoming(self, obj): - if isinstance(obj, dict): - value = {} - for k, v in obj.items(): - value[self.replace_key(k)] = self.replace_incoming(v) - elif isinstance(obj, list): - value = [self.replace_incoming(elem) - for elem in obj] - else: - value = obj - - return value - - def replace_outgoing(self, obj): - if isinstance(obj, dict): - value = {} - for k, v in obj.items(): - value[self.revert_key(k)] = self.replace_outgoing(v) - elif isinstance(obj, list): - value = [self.replace_outgoing(elem) - for elem in obj] - else: - value = obj - - return value - - def transform_incoming(self, son, collection): - return self.replace_incoming(son) - - def transform_outgoing(self, son, collection): - if not collection or collection.name != 'fact': - return son - return self.replace_outgoing(son) - -def register_key_transform(db): - #db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')])) - pass diff --git a/awx/main/migrations/0005_v300_migrate_facts.py b/awx/main/migrations/0005_v300_migrate_facts.py index 8362227c2f..058c5970df 100644 --- a/awx/main/migrations/0005_v300_migrate_facts.py +++ b/awx/main/migrations/0005_v300_migrate_facts.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- from __future__ import unicode_literals -from awx.main.migrations import _system_tracking as system_tracking from django.db import migrations class Migration(migrations.Migration): @@ -11,5 +10,4 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(system_tracking.migrate_facts), ] diff --git a/awx/main/migrations/_system_tracking.py b/awx/main/migrations/_system_tracking.py deleted file mode 100644 index 931c5c467c..0000000000 --- a/awx/main/migrations/_system_tracking.py +++ /dev/null @@ -1,52 +0,0 @@ - -import logging - -from django.utils.encoding import smart_text -from django.conf import settings - -from awx.fact.models import FactVersion -from awx.fact.utils.dbtransform import KeyTransform -from mongoengine.connection import ConnectionError -from pymongo.errors import OperationFailure - -logger = logging.getLogger('system_tracking_migrations') - -def migrate_facts(apps, schema_editor): - Fact = apps.get_model('main', "Fact") - Host = apps.get_model('main', "Host") - - if (not hasattr(settings, 'MONGO_HOST')) or settings.MONGO_HOST == NotImplemented: - logger.info("failed to find MONGO_HOST in settings. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.") - # If settings do not specify a mongo database, do not raise error or drop db - return (0, 0) - - try: - n = FactVersion.objects.all().count() - except ConnectionError: - # Let the user know about the error. Likely this is - # a new install and we just don't need to do this - logger.info(smart_text(u"failed to connect to mongo database host {}. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.".format(settings.MONGO_HOST))) - return (0, 0) - except OperationFailure: - # The database was up but something happened when we tried to query it - logger.info(smart_text(u"failed to connect to issue Mongo query on host {}. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.".format(settings.MONGO_HOST))) - return (0, 0) - - migrated_count = 0 - not_migrated_count = 0 - transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) - for factver in FactVersion.objects.all().no_cache(): - try: - host = Host.objects.only('id').get(inventory__id=factver.host.inventory_id, name=factver.host.hostname) - fact_obj = transform.replace_outgoing(factver.fact) - Fact.objects.create(host_id=host.id, timestamp=fact_obj.timestamp, module=fact_obj.module, facts=fact_obj.fact).save() - migrated_count += 1 - except Host.DoesNotExist: - # No host was found to migrate the facts to. - # This isn't a hard error. Just something the user would want to know. - logger.info(smart_text(u"unable to migrate fact {} not found in Postgres <{}, {}>".format(factver.id, factver.host.inventory_id, factver.host.hostname))) - not_migrated_count += 1 - - logger.info(smart_text(u"successfully migrated {} records of system_tracking data from Mongo to Postgres. {} records not migrated due to corresponding pairs not found in Postgres.".format(migrated_count, not_migrated_count))) - return (migrated_count, not_migrated_count) - diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 9ede3803a9..714a2afc6d 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -210,7 +210,6 @@ INSTALLED_APPS = ( 'awx.main', 'awx.api', 'awx.ui', - 'awx.fact', 'awx.sso', 'solo', ) diff --git a/awx/settings/development.py b/awx/settings/development.py index 6c7f8465a2..f2d72a1113 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -54,7 +54,7 @@ PENDO_TRACKING_STATE = "off" try: import django_jenkins INSTALLED_APPS += (django_jenkins.__name__,) - PROJECT_APPS = ('awx.main.tests', 'awx.api.tests', 'awx.fact.tests',) + PROJECT_APPS = ('awx.main.tests', 'awx.api.tests',) except ImportError: pass diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 4c97f11186..2379e46633 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -62,7 +62,6 @@ lxml==3.4.4 Markdown==2.4.1 M2Crypto==0.22.3 mock==1.0.1 -mongoengine==0.9.0 monotonic==0.6 msgpack-python==0.4.7 munch==2.0.4 @@ -89,7 +88,6 @@ pycrypto==2.6.1 pycparser==2.14 pygerduty==0.32.1 PyJWT==1.4.0 -pymongo==2.8 pyOpenSSL==16.0.0 pyparsing==2.0.7 pyrad==2.0