From 1425718d9e0c17df6259db0bb0bdec640c08ab0a Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 2 Oct 2015 15:57:50 -0400 Subject: [PATCH 1/4] Improve the efficiency of mongo connections - Remove the connect action in the fact __init__ module - Monkeypatch mongoengine.Document's _get_db to force a new connection or raise an error if we don't have the proper settings. --- awx/fact/__init__.py | 26 -------------------------- awx/fact/models/fact.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py index db6bd7fb41..d9f8f41d12 100644 --- a/awx/fact/__init__.py +++ b/awx/fact/__init__.py @@ -1,31 +1,5 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved. -from __future__ import absolute_import - import logging -from django.conf import settings - -from mongoengine import connect -from mongoengine.connection import get_db, ConnectionError -from .utils.dbtransform import register_key_transform - logger = logging.getLogger('awx.fact') - -# Connect to Mongo -try: - # Sanity check: If we have intentionally invalid settings, then we - # know we cannot connect. - if settings.MONGO_HOST == NotImplemented: - raise ConnectionError - - # Attempt to connect to the MongoDB database. - connect(settings.MONGO_DB, - host=settings.MONGO_HOST, - port=int(settings.MONGO_PORT), - username=settings.MONGO_USERNAME, - password=settings.MONGO_PASSWORD, - tz_aware=settings.USE_TZ) - register_key_transform(get_db()) -except ConnectionError: - logger.info('Failed to establish connect to MongoDB') diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index e6efb9d782..0342ee94ba 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -1,12 +1,47 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved +from mongoengine import connect from mongoengine.base import BaseField from mongoengine import Document, DateTimeField, ReferenceField, StringField, IntField +from mongoengine.connection import get_db, ConnectionError +from awx.fact.utils.dbtransform import register_key_transform, KeyTransform from awx.fact.utils.dbtransform import KeyTransform +from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME + +from django.conf import settings + +import logging +logger = logging.getLogger('awx.fact.models.fact') + + key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) +@classmethod +def _get_db_monkeypatched(cls): + """ Override the default _get_db mechanism to start a connection to the database """ + # Connect to Mongo + try: + # Sanity check: If we have intentionally invalid settings, then we + # know we cannot connect. + if settings.MONGO_HOST == NotImplemented: + raise ConnectionError + + # Attempt to connect to the MongoDB database. + connect(settings.MONGO_DB, + host=settings.MONGO_HOST, + port=int(settings.MONGO_PORT), + username=settings.MONGO_USERNAME, + password=settings.MONGO_PASSWORD, + tz_aware=settings.USE_TZ) + register_key_transform(get_db()) + except ConnectionError: + logger.info('Failed to establish connect to MongoDB') + return get_db(cls._meta.get("db_alias", "default")) + +Document._get_db = _get_db_monkeypatched + class TransformField(BaseField): def to_python(self, value): return key_transform.transform_outgoing(value, None) From c65d068d5531ea979c86eacc047326b87f7f227c Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 2 Oct 2015 16:04:47 -0400 Subject: [PATCH 2/4] Add note about wanting to use register_connection I think this would be the cleaner way to go about things but there are some unanswered questions about that route. --- awx/fact/models/fact.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index 0342ee94ba..ba88104e55 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -18,6 +18,8 @@ logger = logging.getLogger('awx.fact.models.fact') key_transform = KeyTransform([('.', '\uff0E'), ('$', '\uff04')]) +# NOTE: I think it might be better to use register_connection here: https://github.com/MongoEngine/mongoengine/blob/0.9/mongoengine/connection.py#L21 +# but I'm not doing that because I don't see how we can also register the key transform as needed or set the tz_aware preference @classmethod def _get_db_monkeypatched(cls): """ Override the default _get_db mechanism to start a connection to the database """ From ac37667649c3599460c07b9d37a94500799d324f Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 2 Oct 2015 16:08:02 -0400 Subject: [PATCH 3/4] Clean up fact module imports --- awx/fact/models/fact.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/awx/fact/models/fact.py b/awx/fact/models/fact.py index ba88104e55..87cc3e1671 100644 --- a/awx/fact/models/fact.py +++ b/awx/fact/models/fact.py @@ -6,9 +6,6 @@ from mongoengine.base import BaseField from mongoengine import Document, DateTimeField, ReferenceField, StringField, IntField from mongoengine.connection import get_db, ConnectionError from awx.fact.utils.dbtransform import register_key_transform, KeyTransform -from awx.fact.utils.dbtransform import KeyTransform - -from mongoengine.connection import get_db, DEFAULT_CONNECTION_NAME from django.conf import settings From b06f7fb883e3e5dd03aa86e2ad8646f1ed907ce1 Mon Sep 17 00:00:00 2001 From: Matthew Jones Date: Fri, 2 Oct 2015 16:56:13 -0400 Subject: [PATCH 4/4] Remove logging from fact init --- awx/fact/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/awx/fact/__init__.py b/awx/fact/__init__.py index d9f8f41d12..e484e62be1 100644 --- a/awx/fact/__init__.py +++ b/awx/fact/__init__.py @@ -1,5 +1,2 @@ # Copyright (c) 2015 Ansible, Inc. # All Rights Reserved. - -import logging -logger = logging.getLogger('awx.fact')