From 9bd0ea5d8da455b345b528df104e2e72ea559880 Mon Sep 17 00:00:00 2001 From: Chris Meyers Date: Mon, 18 May 2015 10:31:36 -0400 Subject: [PATCH] add module fact cleanup granularity --- awx/main/management/commands/cleanup_facts.py | 16 +++++-- awx/main/tests/commands/base.py | 23 ---------- awx/main/tests/commands/cleanup_facts.py | 46 ++++++++++++------- 3 files changed, 41 insertions(+), 44 deletions(-) diff --git a/awx/main/management/commands/cleanup_facts.py b/awx/main/management/commands/cleanup_facts.py index 81387636b6..0cdc248ad9 100644 --- a/awx/main/management/commands/cleanup_facts.py +++ b/awx/main/management/commands/cleanup_facts.py @@ -29,7 +29,7 @@ class CleanupFacts(object): # # pivot -= granularity # group by host - def cleanup(self, older_than_abs, granularity): + def cleanup(self, older_than_abs, granularity, module=None): flag_delete_all = False fact_oldest = FactVersion.objects.all().order_by('timestamp').first() if not fact_oldest: @@ -49,6 +49,8 @@ class CleanupFacts(object): } if not flag_delete_all: kv['timestamp__gt'] = date_pivot_next + if module: + kv['module'] = module version_objs = FactVersion.objects.filter(**kv).order_by('-timestamp') @@ -84,9 +86,9 @@ class CleanupFacts(object): ''' older_than and granularity are of type relativedelta ''' - def run(self, older_than, granularity): + def run(self, older_than, granularity, module=None): t = now() - deleted_count = self.cleanup(t - older_than, granularity) + deleted_count = self.cleanup(t - older_than, granularity, module=module) print("Deleted %d facts." % deleted_count) class Command(BaseCommand): @@ -99,7 +101,11 @@ class Command(BaseCommand): make_option('--granularity', dest='granularity', default=None, - help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'),) + help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'), + make_option('--module', + dest='module', + default=None, + help='Limit cleanup to a particular module.'),) def __init__(self): super(Command, self).__init__() @@ -135,5 +141,5 @@ class Command(BaseCommand): if granularity is None: raise CommandError('--granularity invalid value "%s"' % options[GRANULARITY]) - cleanup_facts.run(older_than, granularity) + cleanup_facts.run(older_than, granularity, module=options['module']) diff --git a/awx/main/tests/commands/base.py b/awx/main/tests/commands/base.py index b8e9434f25..2f1d795d27 100644 --- a/awx/main/tests/commands/base.py +++ b/awx/main/tests/commands/base.py @@ -5,7 +5,6 @@ import StringIO import sys import json -from dateutil.relativedelta import relativedelta # Django from django.core.management import call_command @@ -13,30 +12,8 @@ from django.core.management import call_command # AWX from awx.main.models import * # noqa from awx.main.tests.base import BaseTestMixin -from awx.fact.models import * # noqa class BaseCommandMixin(BaseTestMixin): - ''' - Base class for tests that run management commands. - ''' - def create_hosts_and_facts(self, epoch, host_count, facts_per_host): - self.hosts = [] - for i in range(0, host_count): - host = FactHost.objects.create(hostname='host_%d' % i) - self.hosts.append(host) - - self.fact = { - 'hello': 'world' - } - self.facts = [] - self.versions = [] - for host in self.hosts: - for i in range(0, facts_per_host): - t = epoch - relativedelta(years=i) - (fact_obj, version_obj) = Fact.add_fact(t, self.fact, host, 'packages') - self.facts.append(fact_obj) - self.versions.append(version_obj) - def create_test_inventories(self): self.setup_users() self.organizations = self.make_organizations(self.super_django_user, 2) diff --git a/awx/main/tests/commands/cleanup_facts.py b/awx/main/tests/commands/cleanup_facts.py index cdddce9f8f..f93bb91fee 100644 --- a/awx/main/tests/commands/cleanup_facts.py +++ b/awx/main/tests/commands/cleanup_facts.py @@ -11,7 +11,7 @@ from django.core.management.base import CommandError # AWX from awx.main.tests.base import BaseTest -from awx.fact.tests.base import MongoDBRequired +from awx.fact.tests.base import MongoDBRequired, FactScanBuilder, TEST_FACT_PACKAGES, TEST_FACT_ANSIBLE, TEST_FACT_SERVICES from awx.main.tests.commands.base import BaseCommandMixin from awx.main.management.commands.cleanup_facts import Command, CleanupFacts from awx.fact.models.fact import * # noqa @@ -19,8 +19,14 @@ from awx.fact.models.fact import * # noqa __all__ = ['CommandTest','CleanupFactsUnitTest', 'CleanupFactsCommandFunctionalTest'] class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired): + def setUp(self): + super(CleanupFactsCommandFunctionalTest, self).setUp() + self.builder = FactScanBuilder() + self.builder.add_fact('ansible', TEST_FACT_ANSIBLE) + def test_invoke_zero_ok(self): - self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20) + self.builder.set_epoch(datetime(year=2015, day=2, month=1, microsecond=0)) + self.builder.build(scan_count=20, host_count=10) result, stdout, stderr = self.run_command('cleanup_facts', granularity='2y', older_than='1d') self.assertEqual(stdout, 'Deleted %s facts.\n' % ((200 / 2))) @@ -30,7 +36,7 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi self.assertEqual(stdout, 'Deleted 0 facts.\n') def test_invoke_all_deleted(self): - self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20) + self.builder.build(scan_count=20, host_count=10) result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d') self.assertEqual(stdout, 'Deleted 200 facts.\n') @@ -40,6 +46,14 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi self.assertIsInstance(result, CommandError) self.assertEqual(str(result), 'Both --granularity and --older_than are required.') + def test_module(self): + self.builder.add_fact('packages', TEST_FACT_PACKAGES) + self.builder.add_fact('services', TEST_FACT_SERVICES) + self.builder.build(scan_count=5, host_count=5) + + result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d', module='packages') + self.assertEqual(stdout, 'Deleted 25 facts.\n') + class CommandTest(BaseTest): @mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run') def test_parameters_ok(self, run): @@ -47,10 +61,11 @@ class CommandTest(BaseTest): kv = { 'older_than': '1d', 'granularity': '1d', + 'module': None, } cmd = Command() cmd.handle(None, **kv) - run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1)) + run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1), module=None) def test_string_time_to_timestamp_ok(self): kvs = [ @@ -132,11 +147,9 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired): def setUp(self): super(CleanupFactsUnitTest, self).setUp() - self.datetime_base = datetime(year=2015, day=2, month=1, microsecond=0) - self.HOSTS = 10 - self.FACTS_PER_HOST = 20 - - self.create_hosts_and_facts(self.datetime_base, self.HOSTS, self.FACTS_PER_HOST) + self.builder = FactScanBuilder() + self.builder.add_fact('ansible', TEST_FACT_ANSIBLE) + self.builder.build(scan_count=20, host_count=10) ''' Create 10 hosts with 20 facts each. A single fact a year for 20 years. @@ -148,20 +161,20 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired): fact_oldest = FactVersion.objects.all().order_by('timestamp').first() granularity = relativedelta(years=2) - deleted_count = cleanup_facts.cleanup(self.datetime_base, granularity) - self.assertEqual(deleted_count, (self.FACTS_PER_HOST * self.HOSTS) / 2) + deleted_count = cleanup_facts.cleanup(self.builder.get_timestamp(0), granularity) + self.assertEqual(deleted_count, (self.builder.get_scan_count() * self.builder.get_host_count()) / 2) # Check the number of facts per host - for host in self.hosts: + for host in self.builder.get_hosts(): count = FactVersion.objects.filter(host=host).count() - self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of FactVersion per host for host %s") + self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of FactVersion per host for host %s") count = Fact.objects.filter(host=host).count() - self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of Fact per host") + self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of Fact per host") # Ensure that only 1 fact exists per granularity time - date_pivot = self.datetime_base - for host in self.hosts: + date_pivot = self.builder.get_timestamp(0) + for host in self.builder.get_hosts(): while date_pivot > fact_oldest.timestamp: date_pivot_next = date_pivot - granularity kv = { @@ -177,3 +190,4 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired): +