External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-11 18:09:57 -03:30
Code Issues Packages Projects Releases Wiki Activity

remove deprecated mongo fact code and deps

Browse Source 
* We have the requirements that users with Tower < 3.0.x must upgrade to
3.0.x before > 3.0.x. Thus, we are save to delete all mongo migration
code and deps.
This commit is contained in:
Chris Meyers 2016-11-09 16:25:27 -05:00
parent 766e4af38e
commit 5d205f1e1b
11 changed files with 1 additions and 345 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
awx/fact/__init__.py
View File

@ -1,2 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.

2
awx/fact/management/__init__.py
View File

@ -1,2 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved

6
awx/fact/models/__init__.py
View File

@ -1,6 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
from __future__ import absolute_import
from .fact import * # noqa

217
awx/fact/models/fact.py
View File

@ -1,217 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
from mongoengine import connect
from mongoengine.base import BaseField
from mongoengine import Document, DateTimeField, ReferenceField, StringField, IntField
from mongoengine.connection import get_db, ConnectionError
from awx.fact.utils.dbtransform import register_key_transform, KeyTransform
from django.conf import settings
import logging
logger = logging.getLogger('awx.fact.models.fact')
key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')])
# NOTE: I think it might be better to use register_connection here: https://github.com/MongoEngine/mongoengine/blob/0.9/mongoengine/connection.py#L21
# but I'm not doing that because I don't see how we can also register the key transform as needed or set the tz_aware preference
@classmethod
def _get_db_monkeypatched(cls):
""" Override the default _get_db mechanism to start a connection to the database """
# Connect to Mongo
try:
# Sanity check: If we have intentionally invalid settings, then we
# know we cannot connect.
if settings.MONGO_HOST == NotImplemented:
raise ConnectionError
# Attempt to connect to the MongoDB database.
connect(settings.MONGO_DB,
host=settings.MONGO_HOST,
port=int(settings.MONGO_PORT),
username=settings.MONGO_USERNAME,
password=settings.MONGO_PASSWORD,
tz_aware=settings.USE_TZ)
register_key_transform(get_db())
except (ConnectionError, AttributeError):
logger.info('Failed to establish connect to MongoDB')
return get_db(cls._meta.get("db_alias", "default"))
Document._get_db = _get_db_monkeypatched
class TransformField(BaseField):
def to_python(self, value):
return key_transform.transform_outgoing(value, None)
def prepare_query_value(self, op, value):
if op == 'set':
value = key_transform.transform_incoming(value, None)
return super(TransformField, self).prepare_query_value(op, value)
def to_mongo(self, value):
value = key_transform.transform_incoming(value, None)
return value
class FactHost(Document):
hostname = StringField(max_length=100, required=True, unique_with='inventory_id')
inventory_id = IntField(required=True, unique_with='hostname')
# TODO: Consider using hashed index on hostname. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
('hostname', 'inventory_id')
]
}
class Fact(Document):
timestamp = DateTimeField(required=True)
host = ReferenceField(FactHost, required=True)
module = StringField(max_length=50, required=True)
fact = TransformField(required=True)
# TODO: Consider using hashed index on host. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
'-timestamp',
'host',
]
}
@staticmethod
def add_fact(timestamp, fact, host, module):
fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact)
fact_obj.save()
version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj)
version_obj.save()
return (fact_obj, version_obj)
# TODO: if we want to relax the need to include module...
# If module not specified then filter query may return more than 1 result.
# Thus, the resulting facts must somehow be unioned/concated/ or kept as an array.
@staticmethod
def get_host_version(hostname, inventory_id, timestamp, module):
try:
host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id)
except FactHost.DoesNotExist:
return None
kv = {
'host' : host.id,
'timestamp__lte': timestamp,
'module': module,
}
try:
facts = Fact.objects.filter(**kv).order_by("-timestamp")
if not facts:
return None
return facts[0]
except Fact.DoesNotExist:
return None
@staticmethod
def get_host_timeline(hostname, inventory_id, module):
try:
host = FactHost.objects.get(hostname=hostname, inventory_id=inventory_id)
except FactHost.DoesNotExist:
return None
kv = {
'host': host.id,
'module': module,
}
return FactVersion.objects.filter(**kv).order_by("-timestamp").values_list('timestamp')
# FIXME: single facts no longer works with the addition of the inventory_id field to the FactHost document
@staticmethod
def get_single_facts(hostnames, fact_key, fact_value, timestamp, module):
kv = {
'hostname': {
'$in': hostnames,
}
}
fields = {
'_id': 1
}
host_ids = FactHost._get_collection().find(kv, fields)
if not host_ids or host_ids.count() == 0:
return None
# TODO: use mongo to transform [{_id: <>}, {_id: <>},...] into [_id, _id,...]
host_ids = [e['_id'] for e in host_ids]
pipeline = []
match = {
'host': {
'$in': host_ids
},
'timestamp': {
'$lte': timestamp
},
'module': module
}
sort = {
'timestamp': -1
}
group = {
'_id': '$host',
'timestamp': {
'$first': '$timestamp'
},
'fact': {
'$first': '$fact'
}
}
project = {
'_id': 0,
'fact': 1,
}
pipeline.append({'$match': match}) # noqa
pipeline.append({'$sort': sort}) # noqa
pipeline.append({'$group': group}) # noqa
pipeline.append({'$project': project}) # noqa
q = FactVersion._get_collection().aggregate(pipeline)
if not q or 'result' not in q or len(q['result']) == 0:
return None
# TODO: use mongo to transform [{fact: <>}, {fact: <>},...] into [fact, fact,...]
fact_ids = [fact['fact'] for fact in q['result']]
kv = {
'fact.%s' % fact_key : fact_value,
'_id': {
'$in': fact_ids
}
}
fields = {
'fact.%s.$' % fact_key : 1,
'host': 1,
'timestamp': 1,
'module': 1,
}
facts = Fact._get_collection().find(kv, fields)
#fact_objs = [Fact(**f) for f in facts]
# Translate pymongo python structure to mongoengine Fact object
fact_objs = []
for f in facts:
f['id'] = f.pop('_id')
fact_objs.append(Fact(**f))
return fact_objs
class FactVersion(Document):
timestamp = DateTimeField(required=True)
host = ReferenceField(FactHost, required=True)
module = StringField(max_length=50, required=True)
fact = ReferenceField(Fact, required=True)
# TODO: Consider using hashed index on module. django-mongo may not support this but
# executing raw js will
meta = {
'indexes': [
'-timestamp',
'module',
'host',
]
}

2
awx/fact/utils/__init__.py
View File

@ -1,2 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved

58
awx/fact/utils/dbtransform.py
View File

@ -1,58 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
# Pymongo
from pymongo.son_manipulator import SONManipulator
class KeyTransform(SONManipulator):
def __init__(self, replace):
self.replace = replace
def replace_key(self, key):
for (replace, replacement) in self.replace:
key = key.replace(replace, replacement)
return key
def revert_key(self, key):
for (replacement, replace) in self.replace:
key = key.replace(replace, replacement)
return key
def replace_incoming(self, obj):
if isinstance(obj, dict):
value = {}
for k, v in obj.items():
value[self.replace_key(k)] = self.replace_incoming(v)
elif isinstance(obj, list):
value = [self.replace_incoming(elem)
for elem in obj]
else:
value = obj
return value
def replace_outgoing(self, obj):
if isinstance(obj, dict):
value = {}
for k, v in obj.items():
value[self.revert_key(k)] = self.replace_outgoing(v)
elif isinstance(obj, list):
value = [self.replace_outgoing(elem)
for elem in obj]
else:
value = obj
return value
def transform_incoming(self, son, collection):
return self.replace_incoming(son)
def transform_outgoing(self, son, collection):
if not collection or collection.name != 'fact':
return son
return self.replace_outgoing(son)
def register_key_transform(db):
#db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')]))
pass

2
awx/main/migrations/0005_v300_migrate_facts.py
View File

@ -1,7 +1,6 @@
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from awx.main.migrations import _system_tracking as system_tracking
from django.db import migrations
class Migration(migrations.Migration):
@ -11,5 +10,4 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(system_tracking.migrate_facts),
]

52
awx/main/migrations/_system_tracking.py
View File

@ -1,52 +0,0 @@
import logging
from django.utils.encoding import smart_text
from django.conf import settings
from awx.fact.models import FactVersion
from awx.fact.utils.dbtransform import KeyTransform
from mongoengine.connection import ConnectionError
from pymongo.errors import OperationFailure
logger = logging.getLogger('system_tracking_migrations')
def migrate_facts(apps, schema_editor):
Fact = apps.get_model('main', "Fact")
Host = apps.get_model('main', "Host")
if (not hasattr(settings, 'MONGO_HOST')) or settings.MONGO_HOST == NotImplemented:
logger.info("failed to find MONGO_HOST in settings. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.")
# If settings do not specify a mongo database, do not raise error or drop db
return (0, 0)
try:
n = FactVersion.objects.all().count()
except ConnectionError:
# Let the user know about the error. Likely this is
# a new install and we just don't need to do this
logger.info(smart_text(u"failed to connect to mongo database host {}. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.".format(settings.MONGO_HOST)))
return (0, 0)
except OperationFailure:
# The database was up but something happened when we tried to query it
logger.info(smart_text(u"failed to connect to issue Mongo query on host {}. Will NOT attempt to migrate system_tracking data from Mongo to Postgres.".format(settings.MONGO_HOST)))
return (0, 0)
migrated_count = 0
not_migrated_count = 0
transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')])
for factver in FactVersion.objects.all().no_cache():
try:
host = Host.objects.only('id').get(inventory__id=factver.host.inventory_id, name=factver.host.hostname)
fact_obj = transform.replace_outgoing(factver.fact)
Fact.objects.create(host_id=host.id, timestamp=fact_obj.timestamp, module=fact_obj.module, facts=fact_obj.fact).save()
migrated_count += 1
except Host.DoesNotExist:
# No host was found to migrate the facts to.
# This isn't a hard error. Just something the user would want to know.
logger.info(smart_text(u"unable to migrate fact {} <inventory, hostname> not found in Postgres <{}, {}>".format(factver.id, factver.host.inventory_id, factver.host.hostname)))
not_migrated_count += 1
logger.info(smart_text(u"successfully migrated {} records of system_tracking data from Mongo to Postgres. {} records not migrated due to corresponding <inventory, hostname> pairs not found in Postgres.".format(migrated_count, not_migrated_count)))
return (migrated_count, not_migrated_count)

1
awx/settings/defaults.py
View File

@ -210,7 +210,6 @@ INSTALLED_APPS = (
'awx.main',
'awx.api',
'awx.ui',
'awx.fact',
'awx.sso',
'solo',
)

2
awx/settings/development.py
View File

@ -54,7 +54,7 @@ PENDO_TRACKING_STATE = "off"
try:
import django_jenkins
INSTALLED_APPS += (django_jenkins.__name__,)
PROJECT_APPS = ('awx.main.tests', 'awx.api.tests', 'awx.fact.tests',)
PROJECT_APPS = ('awx.main.tests', 'awx.api.tests',)
except ImportError:
pass

2
requirements/requirements.txt
View File

@ -62,7 +62,6 @@ lxml==3.4.4
Markdown==2.4.1
M2Crypto==0.22.3
mock==1.0.1
mongoengine==0.9.0
monotonic==0.6
msgpack-python==0.4.7
munch==2.0.4
@ -89,7 +88,6 @@ pycrypto==2.6.1
pycparser==2.14
pygerduty==0.32.1
PyJWT==1.4.0
pymongo==2.8
pyOpenSSL==16.0.0
pyparsing==2.0.7
pyrad==2.0