External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-13 02:50:02 -03:30
Code Issues Packages Projects Releases Wiki Activity

add module fact cleanup granularity

Browse Source
This commit is contained in:
Chris Meyers 2015-05-18 10:31:36 -04:00
parent 11e7d96191
commit 9bd0ea5d8d
3 changed files with 41 additions and 44 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
awx/main/management/commands/cleanup_facts.py
View File

@ -29,7 +29,7 @@ class CleanupFacts(object):
#
# pivot -= granularity
# group by host
def cleanup(self, older_than_abs, granularity):
def cleanup(self, older_than_abs, granularity, module=None):
flag_delete_all = False
fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
if not fact_oldest:
@ -49,6 +49,8 @@ class CleanupFacts(object):
}
if not flag_delete_all:
kv['timestamp__gt'] = date_pivot_next
if module:
kv['module'] = module
version_objs = FactVersion.objects.filter(**kv).order_by('-timestamp')
@ -84,9 +86,9 @@ class CleanupFacts(object):
'''
older_than and granularity are of type relativedelta
'''
def run(self, older_than, granularity):
def run(self, older_than, granularity, module=None):
t = now()
deleted_count = self.cleanup(t - older_than, granularity)
deleted_count = self.cleanup(t - older_than, granularity, module=module)
print("Deleted %d facts." % deleted_count)
class Command(BaseCommand):
@ -99,7 +101,11 @@ class Command(BaseCommand):
make_option('--granularity',
dest='granularity',
default=None,
help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'),)
help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'),
make_option('--module',
dest='module',
default=None,
help='Limit cleanup to a particular module.'),)
def __init__(self):
super(Command, self).__init__()
@ -135,5 +141,5 @@ class Command(BaseCommand):
if granularity is None:
raise CommandError('--granularity invalid value "%s"' % options[GRANULARITY])
cleanup_facts.run(older_than, granularity)
cleanup_facts.run(older_than, granularity, module=options['module'])

23
awx/main/tests/commands/base.py
View File

@ -5,7 +5,6 @@
import StringIO
import sys
import json
from dateutil.relativedelta import relativedelta
# Django
from django.core.management import call_command
@ -13,30 +12,8 @@ from django.core.management import call_command
# AWX
from awx.main.models import * # noqa
from awx.main.tests.base import BaseTestMixin
from awx.fact.models import * # noqa
class BaseCommandMixin(BaseTestMixin):
'''
Base class for tests that run management commands.
'''
def create_hosts_and_facts(self, epoch, host_count, facts_per_host):
self.hosts = []
for i in range(0, host_count):
host = FactHost.objects.create(hostname='host_%d' % i)
self.hosts.append(host)
self.fact = {
'hello': 'world'
}
self.facts = []
self.versions = []
for host in self.hosts:
for i in range(0, facts_per_host):
t = epoch - relativedelta(years=i)
(fact_obj, version_obj) = Fact.add_fact(t, self.fact, host, 'packages')
self.facts.append(fact_obj)
self.versions.append(version_obj)
def create_test_inventories(self):
self.setup_users()
self.organizations = self.make_organizations(self.super_django_user, 2)

46
awx/main/tests/commands/cleanup_facts.py
View File

@ -11,7 +11,7 @@ from django.core.management.base import CommandError
# AWX
from awx.main.tests.base import BaseTest
from awx.fact.tests.base import MongoDBRequired
from awx.fact.tests.base import MongoDBRequired, FactScanBuilder, TEST_FACT_PACKAGES, TEST_FACT_ANSIBLE, TEST_FACT_SERVICES
from awx.main.tests.commands.base import BaseCommandMixin
from awx.main.management.commands.cleanup_facts import Command, CleanupFacts
from awx.fact.models.fact import * # noqa
@ -19,8 +19,14 @@ from awx.fact.models.fact import * # noqa
__all__ = ['CommandTest','CleanupFactsUnitTest', 'CleanupFactsCommandFunctionalTest']
class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self):
super(CleanupFactsCommandFunctionalTest, self).setUp()
self.builder = FactScanBuilder()
self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
def test_invoke_zero_ok(self):
self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20)
self.builder.set_epoch(datetime(year=2015, day=2, month=1, microsecond=0))
self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='2y', older_than='1d')
self.assertEqual(stdout, 'Deleted %s facts.\n' % ((200 / 2)))
@ -30,7 +36,7 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi
self.assertEqual(stdout, 'Deleted 0 facts.\n')
def test_invoke_all_deleted(self):
self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20)
self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d')
self.assertEqual(stdout, 'Deleted 200 facts.\n')
@ -40,6 +46,14 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi
self.assertIsInstance(result, CommandError)
self.assertEqual(str(result), 'Both --granularity and --older_than are required.')
def test_module(self):
self.builder.add_fact('packages', TEST_FACT_PACKAGES)
self.builder.add_fact('services', TEST_FACT_SERVICES)
self.builder.build(scan_count=5, host_count=5)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d', module='packages')
self.assertEqual(stdout, 'Deleted 25 facts.\n')
class CommandTest(BaseTest):
@mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
def test_parameters_ok(self, run):
@ -47,10 +61,11 @@ class CommandTest(BaseTest):
kv = {
'older_than': '1d',
'granularity': '1d',
'module': None,
}
cmd = Command()
cmd.handle(None, **kv)
run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1))
run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1), module=None)
def test_string_time_to_timestamp_ok(self):
kvs = [
@ -132,11 +147,9 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self):
super(CleanupFactsUnitTest, self).setUp()
self.datetime_base = datetime(year=2015, day=2, month=1, microsecond=0)
self.HOSTS = 10
self.FACTS_PER_HOST = 20
self.create_hosts_and_facts(self.datetime_base, self.HOSTS, self.FACTS_PER_HOST)
self.builder = FactScanBuilder()
self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
self.builder.build(scan_count=20, host_count=10)
'''
Create 10 hosts with 20 facts each. A single fact a year for 20 years.
@ -148,20 +161,20 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):
fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
granularity = relativedelta(years=2)
deleted_count = cleanup_facts.cleanup(self.datetime_base, granularity)
self.assertEqual(deleted_count, (self.FACTS_PER_HOST * self.HOSTS) / 2)
deleted_count = cleanup_facts.cleanup(self.builder.get_timestamp(0), granularity)
self.assertEqual(deleted_count, (self.builder.get_scan_count() * self.builder.get_host_count()) / 2)
# Check the number of facts per host
for host in self.hosts:
for host in self.builder.get_hosts():
count = FactVersion.objects.filter(host=host).count()
self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of FactVersion per host for host %s")
self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of FactVersion per host for host %s")
count = Fact.objects.filter(host=host).count()
self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of Fact per host")
self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of Fact per host")
# Ensure that only 1 fact exists per granularity time
date_pivot = self.datetime_base
for host in self.hosts:
date_pivot = self.builder.get_timestamp(0)
for host in self.builder.get_hosts():
while date_pivot > fact_oldest.timestamp:
date_pivot_next = date_pivot - granularity
kv = {
@ -177,3 +190,4 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):