mirror of
https://github.com/ansible/awx.git
synced 2026-03-26 05:15:02 -02:30
moved new fact implementation to fact app
This commit is contained in:
Chris Meyers
@@ -1,55 +0,0 @@
|
||||
# Copyright (c) 2014, Ansible, Inc.
|
||||
# All Rights Reserved.
|
||||
from pymongo.son_manipulator import SONManipulator
|
||||
|
||||
'''
|
||||
Inspired by: https://stackoverflow.com/questions/8429318/how-to-use-dot-in-field-name/20698802#20698802
|
||||
|
||||
Replace . and $ with unicode values
|
||||
'''
|
||||
class KeyTransform(SONManipulator):
|
||||
def __init__(self, replace):
|
||||
self.replace = replace
|
||||
|
||||
def transform_key(self, key, replace, replacement):
|
||||
"""Transform key for saving to database."""
|
||||
return key.replace(replace, replacement)
|
||||
|
||||
def revert_key(self, key, replace, replacement):
|
||||
"""Restore transformed key returning from database."""
|
||||
return key.replace(replacement, replace)
|
||||
|
||||
def transform_incoming(self, son, collection):
|
||||
"""Recursively replace all keys that need transforming."""
|
||||
for (key, value) in son.items():
|
||||
for r in self.replace:
|
||||
replace = r[0]
|
||||
replacement = r[1]
|
||||
if replace in key:
|
||||
if isinstance(value, dict):
|
||||
son[self.transform_key(key, replace, replacement)] = self.transform_incoming(
|
||||
son.pop(key), collection)
|
||||
else:
|
||||
son[self.transform_key(key, replace, replacement)] = son.pop(key)
|
||||
elif isinstance(value, dict): # recurse into sub-docs
|
||||
son[key] = self.transform_incoming(value, collection)
|
||||
return son
|
||||
|
||||
def transform_outgoing(self, son, collection):
|
||||
"""Recursively restore all transformed keys."""
|
||||
for (key, value) in son.items():
|
||||
for r in self.replace:
|
||||
replace = r[0]
|
||||
replacement = r[1]
|
||||
if replacement in key:
|
||||
if isinstance(value, dict):
|
||||
son[self.revert_key(key, replace, replacement)] = self.transform_outgoing(
|
||||
son.pop(key), collection)
|
||||
else:
|
||||
son[self.revert_key(key, replace, replacement)] = son.pop(key)
|
||||
elif isinstance(value, dict): # recurse into sub-docs
|
||||
son[key] = self.transform_outgoing(value, collection)
|
||||
return son
|
||||
|
||||
def register_key_transform(db):
|
||||
db.add_son_manipulator(KeyTransform([('.', '\uff0E'), ('$', '\uff04')]))
|
||||
@@ -6,7 +6,7 @@ from datetime import datetime
|
||||
|
||||
from django.core.management.base import NoArgsCommand
|
||||
|
||||
from awx.main.models import * # noqa
|
||||
from awx.fact.models.fact import * # noqa
|
||||
from awx.main.socket import Socket
|
||||
|
||||
_MODULES = ['packages', 'services', 'files']
|
||||
|
||||
@@ -16,7 +16,6 @@ from awx.main.models.ad_hoc_commands import * # noqa
|
||||
from awx.main.models.schedules import * # noqa
|
||||
from awx.main.models.activity_stream import * # noqa
|
||||
from awx.main.models.ha import * # noqa
|
||||
from awx.main.models.fact import * # noqa
|
||||
|
||||
# Monkeypatch Django serializer to ignore django-taggit fields (which break
|
||||
# the dumpdata command; see https://github.com/alex/django-taggit/issues/155).
|
||||
|
||||
@@ -1,95 +0,0 @@
|
||||
from mongoengine import Document, DynamicDocument, DateTimeField, ReferenceField, StringField
|
||||
|
||||
class FactHost(Document):
|
||||
hostname = StringField(max_length=100, required=True, unique=True)
|
||||
|
||||
# TODO: Consider using hashed index on hostname. django-mongo may not support this but
|
||||
# executing raw js will
|
||||
meta = {
|
||||
'indexes': [
|
||||
'hostname'
|
||||
]
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def get_host_id(hostname):
|
||||
host = FactHost.objects.get(hostname=hostname)
|
||||
if host:
|
||||
return host.id
|
||||
return None
|
||||
|
||||
class Fact(DynamicDocument):
|
||||
timestamp = DateTimeField(required=True)
|
||||
host = ReferenceField(FactHost, required=True)
|
||||
module = StringField(max_length=50, required=True)
|
||||
# fact = <anything>
|
||||
|
||||
# TODO: Consider using hashed index on host. django-mongo may not support this but
|
||||
# executing raw js will
|
||||
meta = {
|
||||
'indexes': [
|
||||
'-timestamp',
|
||||
'host'
|
||||
]
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def add_fact(timestamp, fact, host, module):
|
||||
fact_obj = Fact(timestamp=timestamp, host=host, module=module, fact=fact)
|
||||
fact_obj.save()
|
||||
version_obj = FactVersion(timestamp=timestamp, host=host, module=module, fact=fact_obj)
|
||||
version_obj.save()
|
||||
return (fact_obj, version_obj)
|
||||
|
||||
# TODO: if we want to relax the need to include module...
|
||||
# If module not specified then filter query may return more than 1 result.
|
||||
# Thus, the resulting facts must somehow be unioned/concated/ or kept as an array.
|
||||
@staticmethod
|
||||
def get_host_version(hostname, timestamp, module):
|
||||
try:
|
||||
host = FactHost.objects.get(hostname=hostname)
|
||||
except FactHost.DoesNotExist:
|
||||
return None
|
||||
|
||||
kv = {
|
||||
'host' : host.id,
|
||||
'timestamp__lte': timestamp,
|
||||
'module': module,
|
||||
}
|
||||
|
||||
try:
|
||||
facts = Fact.objects.filter(**kv)
|
||||
if not facts:
|
||||
return None
|
||||
return facts[0]
|
||||
except Fact.DoesNotExist:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_host_timeline(hostname, module):
|
||||
try:
|
||||
host = FactHost.objects.get(hostname=hostname)
|
||||
except FactHost.DoesNotExist:
|
||||
return None
|
||||
|
||||
kv = {
|
||||
'host': host.id,
|
||||
'module': module,
|
||||
}
|
||||
|
||||
return FactVersion.objects.filter(**kv).values_list('timestamp')
|
||||
|
||||
class FactVersion(Document):
|
||||
timestamp = DateTimeField(required=True)
|
||||
host = ReferenceField(FactHost, required=True)
|
||||
module = StringField(max_length=50, required=True)
|
||||
fact = ReferenceField(Fact, required=True)
|
||||
# TODO: Consider using hashed index on module. django-mongo may not support this but
|
||||
# executing raw js will
|
||||
meta = {
|
||||
'indexes': [
|
||||
'-timestamp',
|
||||
'module'
|
||||
]
|
||||
}
|
||||
|
||||
@@ -15,6 +15,4 @@ from awx.main.tests.activity_stream import * # noqa
|
||||
from awx.main.tests.schedules import * # noqa
|
||||
from awx.main.tests.redact import * # noqa
|
||||
from awx.main.tests.views import * # noqa
|
||||
from awx.main.tests.models import * # noqa
|
||||
from awx.main.tests.commands import * # noqa
|
||||
from awx.main.tests.dbtransform import * # noqa
|
||||
|
||||
@@ -13,7 +13,7 @@ from mock import MagicMock
|
||||
from awx.main.tests.base import BaseTest, MongoDBRequired
|
||||
from awx.main.tests.commands.base import BaseCommandMixin
|
||||
from awx.main.management.commands.run_fact_cache_receiver import FactCacheReceiver
|
||||
from awx.main.models.fact import * # noqa
|
||||
from awx.fact.models.fact import * # noqa
|
||||
|
||||
__all__ = ['RunFactCacheReceiverUnitTest', 'RunFactCacheReceiverFunctionalTest']
|
||||
|
||||
|
||||
@@ -1,77 +0,0 @@
|
||||
# Copyright (c) 2015 Ansible, Inc.
|
||||
# All Rights Reserved
|
||||
|
||||
# Python
|
||||
from datetime import datetime
|
||||
from mongoengine import connect
|
||||
|
||||
# Django
|
||||
from django.conf import settings
|
||||
|
||||
# AWX
|
||||
from awx.main.tests.base import BaseTest, MongoDBRequired
|
||||
from awx.main.models.fact import * # noqa
|
||||
|
||||
__all__ = ['DBTransformTest']
|
||||
|
||||
class DBTransformTest(BaseTest, MongoDBRequired):
|
||||
def setUp(self):
|
||||
super(DBTransformTest, self).setUp()
|
||||
|
||||
# Create a db connection that doesn't have the transformation registered
|
||||
# Note: this goes through pymongo not mongoengine
|
||||
self.client = connect(settings.MONGO_DB)
|
||||
self.db = self.client[settings.MONGO_DB]
|
||||
|
||||
def _create_fact(self):
|
||||
fact = {}
|
||||
fact[self.k] = self.v
|
||||
h = FactHost(hostname='blah')
|
||||
h.save()
|
||||
f = Fact(host=h,module='blah',timestamp=datetime.now(),fact=fact)
|
||||
f.save()
|
||||
return f
|
||||
|
||||
def create_dot_fact(self):
|
||||
self.k = 'this.is.a.key'
|
||||
self.v = 'this.is.a.value'
|
||||
|
||||
self.k_uni = 'this\uff0Eis\uff0Ea\uff0Ekey'
|
||||
|
||||
return self._create_fact()
|
||||
|
||||
def create_dollar_fact(self):
|
||||
self.k = 'this$is$a$key'
|
||||
self.v = 'this$is$a$value'
|
||||
|
||||
self.k_uni = 'this\uff04is\uff04a\uff04key'
|
||||
|
||||
return self._create_fact()
|
||||
|
||||
def check_unicode(self, f):
|
||||
f_raw = self.db.fact.find_one(id=f.id)
|
||||
self.assertIn(self.k_uni, f_raw['fact'])
|
||||
self.assertEqual(f_raw['fact'][self.k_uni], self.v)
|
||||
|
||||
# Ensure key . are being transformed to the equivalent unicode into the database
|
||||
def test_key_transform_dot_unicode_in_storage(self):
|
||||
f = self.create_dot_fact()
|
||||
self.check_unicode(f)
|
||||
|
||||
# Ensure key $ are being transformed to the equivalent unicode into the database
|
||||
def test_key_transform_dollar_unicode_in_storage(self):
|
||||
f = self.create_dollar_fact()
|
||||
self.check_unicode(f)
|
||||
|
||||
def check_transform(self):
|
||||
f = Fact.objects.all()[0]
|
||||
self.assertIn(self.k, f.fact)
|
||||
self.assertEqual(f.fact[self.k], self.v)
|
||||
|
||||
def test_key_transform_dot_on_retreive(self):
|
||||
self.create_dot_fact()
|
||||
self.check_transform()
|
||||
|
||||
def test_key_transform_dollar_on_retreive(self):
|
||||
self.create_dollar_fact()
|
||||
self.check_transform()
|
||||
@@ -1,4 +0,0 @@
|
||||
# Copyright (c) 2015 Ansible, Inc.
|
||||
# All Rights Reserved
|
||||
|
||||
from awx.main.tests.models.fact import * # noqa
|
||||
@@ -1,154 +0,0 @@
|
||||
# Copyright (c) 2015 Ansible, Inc.
|
||||
# All Rights Reserved
|
||||
|
||||
# Python
|
||||
from datetime import datetime
|
||||
from copy import deepcopy
|
||||
|
||||
# Django
|
||||
|
||||
# AWX
|
||||
from awx.main.models.fact import * # noqa
|
||||
from awx.main.tests.base import BaseTest, MongoDBRequired
|
||||
|
||||
__all__ = ['FactHostTest', 'FactTest', 'FactGetHostVersionTest', 'FactGetHostTimeline']
|
||||
|
||||
TEST_FACT_DATA = {
|
||||
'hostname': 'hostname1',
|
||||
'add_fact_data': {
|
||||
'timestamp': datetime.now(),
|
||||
'host': None,
|
||||
'module': 'packages',
|
||||
'fact': {
|
||||
"accountsservice": [
|
||||
{
|
||||
"architecture": "amd64",
|
||||
"name": "accountsservice",
|
||||
"source": "apt",
|
||||
"version": "0.6.35-0ubuntu7.1"
|
||||
}
|
||||
],
|
||||
"acpid": [
|
||||
{
|
||||
"architecture": "amd64",
|
||||
"name": "acpid",
|
||||
"source": "apt",
|
||||
"version": "1:2.0.21-1ubuntu2"
|
||||
}
|
||||
],
|
||||
"adduser": [
|
||||
{
|
||||
"architecture": "all",
|
||||
"name": "adduser",
|
||||
"source": "apt",
|
||||
"version": "3.113+nmu3ubuntu3"
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
}
|
||||
# Strip off microseconds because mongo has less precision
|
||||
TEST_FACT_DATA['add_fact_data']['timestamp'] = TEST_FACT_DATA['add_fact_data']['timestamp'].replace(microsecond=0)
|
||||
|
||||
def create_host_document():
|
||||
TEST_FACT_DATA['add_fact_data']['host'] = FactHost(hostname=TEST_FACT_DATA['hostname']).save()
|
||||
|
||||
def create_fact_scans(count=1):
|
||||
timestamps = []
|
||||
for i in range(0, count):
|
||||
data = deepcopy(TEST_FACT_DATA)
|
||||
t = datetime.now().replace(year=2015 - i, microsecond=0)
|
||||
data['add_fact_data']['timestamp'] = t
|
||||
(f, v) = Fact.add_fact(**data['add_fact_data'])
|
||||
timestamps.append(t)
|
||||
|
||||
return timestamps
|
||||
|
||||
|
||||
class FactHostTest(BaseTest, MongoDBRequired):
|
||||
def test_create_host(self):
|
||||
host = FactHost(hostname=TEST_FACT_DATA['hostname'])
|
||||
host.save()
|
||||
|
||||
host = FactHost.objects.get(hostname=TEST_FACT_DATA['hostname'])
|
||||
self.assertIsNotNone(host, "Host added but not found")
|
||||
self.assertEqual(TEST_FACT_DATA['hostname'], host.hostname, "Gotten record hostname does not match expected hostname")
|
||||
|
||||
# Ensure an error is raised for .get() that doesn't match a record.
|
||||
def test_get_host_id_no_result(self):
|
||||
host = FactHost(hostname=TEST_FACT_DATA['hostname'])
|
||||
host.save()
|
||||
|
||||
self.assertRaises(FactHost.DoesNotExist, FactHost.objects.get, hostname='doesnotexist')
|
||||
|
||||
class FactTest(BaseTest, MongoDBRequired):
|
||||
def setUp(self):
|
||||
super(FactTest, self).setUp()
|
||||
create_host_document()
|
||||
|
||||
def test_add_fact(self):
|
||||
(f_obj, v_obj) = Fact.add_fact(**TEST_FACT_DATA['add_fact_data'])
|
||||
f = Fact.objects.get(id=f_obj.id)
|
||||
v = FactVersion.objects.get(id=v_obj.id)
|
||||
|
||||
self.assertEqual(f.id, f_obj.id)
|
||||
self.assertEqual(f.module, TEST_FACT_DATA['add_fact_data']['module'])
|
||||
self.assertEqual(f.fact, TEST_FACT_DATA['add_fact_data']['fact'])
|
||||
self.assertEqual(f.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
|
||||
|
||||
# host relationship created
|
||||
self.assertEqual(f.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
|
||||
|
||||
# version created and related
|
||||
self.assertEqual(v.id, v_obj.id)
|
||||
self.assertEqual(v.timestamp, TEST_FACT_DATA['add_fact_data']['timestamp'])
|
||||
self.assertEqual(v.host.id, TEST_FACT_DATA['add_fact_data']['host'].id)
|
||||
self.assertEqual(v.fact.id, f_obj.id)
|
||||
self.assertEqual(v.fact.module, TEST_FACT_DATA['add_fact_data']['module'])
|
||||
|
||||
class FactGetHostVersionTest(BaseTest, MongoDBRequired):
|
||||
def setUp(self):
|
||||
super(FactGetHostVersionTest, self).setUp()
|
||||
create_host_document()
|
||||
|
||||
self.t1 = datetime.now().replace(second=1, microsecond=0)
|
||||
self.t2 = datetime.now().replace(second=2, microsecond=0)
|
||||
data = deepcopy(TEST_FACT_DATA)
|
||||
data['add_fact_data']['timestamp'] = self.t1
|
||||
(self.f1, self.v1) = Fact.add_fact(**data['add_fact_data'])
|
||||
data = deepcopy(TEST_FACT_DATA)
|
||||
data['add_fact_data']['timestamp'] = self.t2
|
||||
(self.f2, self.v2) = Fact.add_fact(**data['add_fact_data'])
|
||||
|
||||
def test_get_host_version_exact_timestamp(self):
|
||||
fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=self.t1, module=TEST_FACT_DATA['add_fact_data']['module'])
|
||||
self.assertIsNotNone(fact, "Set of Facts not found")
|
||||
self.assertEqual(self.f1.id, fact.id)
|
||||
self.assertEqual(self.f1.fact, fact.fact)
|
||||
|
||||
def test_get_host_version_lte_timestamp(self):
|
||||
t3 = datetime.now().replace(second=3, microsecond=0)
|
||||
fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module'])
|
||||
self.assertEqual(self.f1.id, fact.id)
|
||||
self.assertEqual(self.f1.fact, fact.fact)
|
||||
|
||||
def test_get_host_version_none(self):
|
||||
t3 = deepcopy(self.t1).replace(second=0)
|
||||
fact = Fact.get_host_version(hostname=TEST_FACT_DATA['hostname'], timestamp=t3, module=TEST_FACT_DATA['add_fact_data']['module'])
|
||||
self.assertIsNone(fact)
|
||||
|
||||
class FactGetHostTimeline(BaseTest, MongoDBRequired):
|
||||
def setUp(self):
|
||||
super(FactGetHostTimeline, self).setUp()
|
||||
create_host_document()
|
||||
|
||||
self.scans = 20
|
||||
self.timestamps = create_fact_scans(self.scans)
|
||||
|
||||
def test_get_host_timeline_ok(self):
|
||||
timestamps = Fact.get_host_timeline(hostname=TEST_FACT_DATA['hostname'], module=TEST_FACT_DATA['add_fact_data']['module'])
|
||||
self.assertIsNotNone(timestamps)
|
||||
self.assertEqual(len(timestamps), len(self.timestamps))
|
||||
for i in range(0, self.scans):
|
||||
self.assertEqual(timestamps[i], self.timestamps[i])
|
||||
|
||||
Reference in New Issue
Block a user