External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-02-21 13:10:11 -03:30
Code Issues Packages Projects Releases Wiki Activity

add module fact cleanup granularity

Browse Source
This commit is contained in:
Chris Meyers
2015-05-18 10:31:36 -04:00
parent 11e7d96191
commit 9bd0ea5d8d
3 changed files with 41 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files

16
awx/main/management/commands/cleanup_facts.py
View File

@@ -29,7 +29,7 @@ class CleanupFacts(object):
# #
# pivot -= granularity # pivot -= granularity
# group by host # group by host
def cleanup(self, older_than_abs, granularity): def cleanup(self, older_than_abs, granularity, module=None):
flag_delete_all = False flag_delete_all = False
fact_oldest = FactVersion.objects.all().order_by('timestamp').first() fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
if not fact_oldest: if not fact_oldest:
@@ -49,6 +49,8 @@ class CleanupFacts(object):
} }
if not flag_delete_all: if not flag_delete_all:
kv['timestamp__gt'] = date_pivot_next kv['timestamp__gt'] = date_pivot_next
if module:
kv['module'] = module
version_objs = FactVersion.objects.filter(**kv).order_by('-timestamp') version_objs = FactVersion.objects.filter(**kv).order_by('-timestamp')
@@ -84,9 +86,9 @@ class CleanupFacts(object):
''' '''
older_than and granularity are of type relativedelta older_than and granularity are of type relativedelta
''' '''
def run(self, older_than, granularity): def run(self, older_than, granularity, module=None):
t = now() t = now()
deleted_count = self.cleanup(t - older_than, granularity) deleted_count = self.cleanup(t - older_than, granularity, module=module)
print("Deleted %d facts." % deleted_count) print("Deleted %d facts." % deleted_count)
class Command(BaseCommand): class Command(BaseCommand):
@@ -99,7 +101,11 @@ class Command(BaseCommand):
make_option('--granularity', make_option('--granularity',
dest='granularity', dest='granularity',
default=None, default=None,
help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'),) help='Window duration to group same hosts by for deletion (w)eek (d)ay or (y)ear (i.e. 5d, 2w, 1y).'),
make_option('--module',
dest='module',
default=None,
help='Limit cleanup to a particular module.'),)
def __init__(self): def __init__(self):
super(Command, self).__init__() super(Command, self).__init__()
@@ -135,5 +141,5 @@ class Command(BaseCommand):
if granularity is None: if granularity is None:
raise CommandError('--granularity invalid value "%s"' % options[GRANULARITY]) raise CommandError('--granularity invalid value "%s"' % options[GRANULARITY])
cleanup_facts.run(older_than, granularity) cleanup_facts.run(older_than, granularity, module=options['module'])

23
awx/main/tests/commands/base.py
View File

@@ -5,7 +5,6 @@
import StringIO import StringIO
import sys import sys
import json import json
from dateutil.relativedelta import relativedelta
# Django # Django
from django.core.management import call_command from django.core.management import call_command
@@ -13,30 +12,8 @@ from django.core.management import call_command
# AWX # AWX
from awx.main.models import * # noqa from awx.main.models import * # noqa
from awx.main.tests.base import BaseTestMixin from awx.main.tests.base import BaseTestMixin
from awx.fact.models import * # noqa
class BaseCommandMixin(BaseTestMixin): class BaseCommandMixin(BaseTestMixin):
'''
Base class for tests that run management commands.
'''
def create_hosts_and_facts(self, epoch, host_count, facts_per_host):
self.hosts = []
for i in range(0, host_count):
host = FactHost.objects.create(hostname='host_%d' % i)
self.hosts.append(host)
self.fact = {
'hello': 'world'
}
self.facts = []
self.versions = []
for host in self.hosts:
for i in range(0, facts_per_host):
t = epoch - relativedelta(years=i)
(fact_obj, version_obj) = Fact.add_fact(t, self.fact, host, 'packages')
self.facts.append(fact_obj)
self.versions.append(version_obj)
def create_test_inventories(self): def create_test_inventories(self):
self.setup_users() self.setup_users()
self.organizations = self.make_organizations(self.super_django_user, 2) self.organizations = self.make_organizations(self.super_django_user, 2)

46
awx/main/tests/commands/cleanup_facts.py
View File

@@ -11,7 +11,7 @@ from django.core.management.base import CommandError
# AWX # AWX
from awx.main.tests.base import BaseTest from awx.main.tests.base import BaseTest
from awx.fact.tests.base import MongoDBRequired from awx.fact.tests.base import MongoDBRequired, FactScanBuilder, TEST_FACT_PACKAGES, TEST_FACT_ANSIBLE, TEST_FACT_SERVICES
from awx.main.tests.commands.base import BaseCommandMixin from awx.main.tests.commands.base import BaseCommandMixin
from awx.main.management.commands.cleanup_facts import Command, CleanupFacts from awx.main.management.commands.cleanup_facts import Command, CleanupFacts
from awx.fact.models.fact import * # noqa from awx.fact.models.fact import * # noqa
@@ -19,8 +19,14 @@ from awx.fact.models.fact import * # noqa
__all__ = ['CommandTest','CleanupFactsUnitTest', 'CleanupFactsCommandFunctionalTest'] __all__ = ['CommandTest','CleanupFactsUnitTest', 'CleanupFactsCommandFunctionalTest']
class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired): class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self):
super(CleanupFactsCommandFunctionalTest, self).setUp()
self.builder = FactScanBuilder()
self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
def test_invoke_zero_ok(self): def test_invoke_zero_ok(self):
self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20) self.builder.set_epoch(datetime(year=2015, day=2, month=1, microsecond=0))
self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='2y', older_than='1d') result, stdout, stderr = self.run_command('cleanup_facts', granularity='2y', older_than='1d')
self.assertEqual(stdout, 'Deleted %s facts.\n' % ((200 / 2))) self.assertEqual(stdout, 'Deleted %s facts.\n' % ((200 / 2)))
@@ -30,7 +36,7 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi
self.assertEqual(stdout, 'Deleted 0 facts.\n') self.assertEqual(stdout, 'Deleted 0 facts.\n')
def test_invoke_all_deleted(self): def test_invoke_all_deleted(self):
self.create_hosts_and_facts(datetime(year=2015, day=2, month=1, microsecond=0), 10, 20) self.builder.build(scan_count=20, host_count=10)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d') result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d')
self.assertEqual(stdout, 'Deleted 200 facts.\n') self.assertEqual(stdout, 'Deleted 200 facts.\n')
@@ -40,6 +46,14 @@ class CleanupFactsCommandFunctionalTest(BaseCommandMixin, BaseTest, MongoDBRequi
self.assertIsInstance(result, CommandError) self.assertIsInstance(result, CommandError)
self.assertEqual(str(result), 'Both --granularity and --older_than are required.') self.assertEqual(str(result), 'Both --granularity and --older_than are required.')
def test_module(self):
self.builder.add_fact('packages', TEST_FACT_PACKAGES)
self.builder.add_fact('services', TEST_FACT_SERVICES)
self.builder.build(scan_count=5, host_count=5)
result, stdout, stderr = self.run_command('cleanup_facts', granularity='0d', older_than='0d', module='packages')
self.assertEqual(stdout, 'Deleted 25 facts.\n')
class CommandTest(BaseTest): class CommandTest(BaseTest):
@mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run') @mock.patch('awx.main.management.commands.cleanup_facts.CleanupFacts.run')
def test_parameters_ok(self, run): def test_parameters_ok(self, run):
@@ -47,10 +61,11 @@ class CommandTest(BaseTest):
kv = { kv = {
'older_than': '1d', 'older_than': '1d',
'granularity': '1d', 'granularity': '1d',
'module': None,
} }
cmd = Command() cmd = Command()
cmd.handle(None, **kv) cmd.handle(None, **kv)
run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1)) run.assert_called_once_with(relativedelta(days=1), relativedelta(days=1), module=None)
def test_string_time_to_timestamp_ok(self): def test_string_time_to_timestamp_ok(self):
kvs = [ kvs = [
@@ -132,11 +147,9 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):
def setUp(self): def setUp(self):
super(CleanupFactsUnitTest, self).setUp() super(CleanupFactsUnitTest, self).setUp()
self.datetime_base = datetime(year=2015, day=2, month=1, microsecond=0) self.builder = FactScanBuilder()
self.HOSTS = 10 self.builder.add_fact('ansible', TEST_FACT_ANSIBLE)
self.FACTS_PER_HOST = 20 self.builder.build(scan_count=20, host_count=10)
self.create_hosts_and_facts(self.datetime_base, self.HOSTS, self.FACTS_PER_HOST)
''' '''
Create 10 hosts with 20 facts each. A single fact a year for 20 years. Create 10 hosts with 20 facts each. A single fact a year for 20 years.
@@ -148,20 +161,20 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):
fact_oldest = FactVersion.objects.all().order_by('timestamp').first() fact_oldest = FactVersion.objects.all().order_by('timestamp').first()
granularity = relativedelta(years=2) granularity = relativedelta(years=2)
deleted_count = cleanup_facts.cleanup(self.datetime_base, granularity) deleted_count = cleanup_facts.cleanup(self.builder.get_timestamp(0), granularity)
self.assertEqual(deleted_count, (self.FACTS_PER_HOST * self.HOSTS) / 2) self.assertEqual(deleted_count, (self.builder.get_scan_count() * self.builder.get_host_count()) / 2)
# Check the number of facts per host # Check the number of facts per host
for host in self.hosts: for host in self.builder.get_hosts():
count = FactVersion.objects.filter(host=host).count() count = FactVersion.objects.filter(host=host).count()
self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of FactVersion per host for host %s") self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of FactVersion per host for host %s")
count = Fact.objects.filter(host=host).count() count = Fact.objects.filter(host=host).count()
self.assertEqual(count, self.FACTS_PER_HOST / 2, "should have half the number of Fact per host") self.assertEqual(count, self.builder.get_scan_count() / 2, "should have half the number of Fact per host")
# Ensure that only 1 fact exists per granularity time # Ensure that only 1 fact exists per granularity time
date_pivot = self.datetime_base date_pivot = self.builder.get_timestamp(0)
for host in self.hosts: for host in self.builder.get_hosts():
while date_pivot > fact_oldest.timestamp: while date_pivot > fact_oldest.timestamp:
date_pivot_next = date_pivot - granularity date_pivot_next = date_pivot - granularity
kv = { kv = {
@@ -177,3 +190,4 @@ class CleanupFactsUnitTest(BaseCommandMixin, BaseTest, MongoDBRequired):