From c1ea48904314c494cf4c613f1882d56f4c8b9010 Mon Sep 17 00:00:00 2001 From: fedora Date: Tue, 13 Apr 2021 19:42:23 -0400 Subject: [PATCH] created a new model 'HostMetrics' which will contain the first timestamp and the most recent timestamp of any automation on a given object and Added a new 'awx-manage' command. command : awx-manage host_metrics --since --until --- awx/main/management/commands/host_metric.py | 54 +++++++++++++++++++ awx/main/migrations/0139_hostmetric.py | 21 ++++++++ awx/main/models/__init__.py | 11 +++- awx/main/models/events.py | 10 ++++ awx/main/models/inventory.py | 6 +++ .../functional/models/test_host_metric.py | 22 ++++++++ 6 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 awx/main/management/commands/host_metric.py create mode 100644 awx/main/migrations/0139_hostmetric.py create mode 100644 awx/main/tests/functional/models/test_host_metric.py diff --git a/awx/main/management/commands/host_metric.py b/awx/main/management/commands/host_metric.py new file mode 100644 index 0000000000..0365cd0d4e --- /dev/null +++ b/awx/main/management/commands/host_metric.py @@ -0,0 +1,54 @@ +from django.core.management.base import BaseCommand +import datetime +from django.core.serializers.json import DjangoJSONEncoder +from awx.main.models.inventory import HostMetric +import json + + +class Command(BaseCommand): + + help = 'This is for offline licensing usage' + + def add_arguments(self, parser): + parser.add_argument('--since', type=datetime.datetime.fromisoformat, help='Start Date in ISO format') + parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format') + parser.add_argument('--json', action='store_true', help='Select output as JSON') + + def handle(self, *args, **options): + since = options.get('since') + until = options.get('until') + + if since is None and until is None: + print("No Arguments received") + return None + + if since is not None and since.tzinfo is None: + since = since.replace(tzinfo=datetime.timezone.utc) + + if until is not None and until.tzinfo is None: + until = until.replace(tzinfo=datetime.timezone.utc) + + filter_kwargs = {} + if since is not None: + filter_kwargs['last_automation__gte'] = since + if until is not None: + filter_kwargs['last_automation__lte'] = until + + result = HostMetric.objects.filter(**filter_kwargs) + + # if --json flag is set, output the result in json format + if options['json']: + list_of_queryset = list(result.values('hostname', 'first_automation', 'last_automation')) + json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder) + print(json_result) + + # --json flag is not set, output in plain text + else: + print(f"Total Number of hosts automated: {len(result)}") + for item in result: + print( + "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format( + hostname=item.hostname, first_automation=item.first_automation, last_automation=item.last_automation + ) + ) + return diff --git a/awx/main/migrations/0139_hostmetric.py b/awx/main/migrations/0139_hostmetric.py new file mode 100644 index 0000000000..6903eb5378 --- /dev/null +++ b/awx/main/migrations/0139_hostmetric.py @@ -0,0 +1,21 @@ +# Generated by Django 2.2.16 on 2021-04-29 16:31 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0138_custom_inventory_scripts_removal'), + ] + + operations = [ + migrations.CreateModel( + name='HostMetric', + fields=[ + ('hostname', models.CharField(max_length=512, primary_key=True, serialize=False)), + ('first_automation', models.DateTimeField(auto_now_add=True, db_index=True, help_text='When the host was first automated against')), + ('last_automation', models.DateTimeField(db_index=True, help_text='When the host was last automated against')), + ], + ), + ] diff --git a/awx/main/models/__init__.py b/awx/main/models/__init__.py index d181222e73..479cecb988 100644 --- a/awx/main/models/__init__.py +++ b/awx/main/models/__init__.py @@ -12,7 +12,16 @@ from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutM from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership # noqa from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env # noqa from awx.main.models.projects import Project, ProjectUpdate # noqa -from awx.main.models.inventory import Group, Host, Inventory, InventorySource, InventoryUpdate, SmartInventoryMembership # noqa +from awx.main.models.inventory import ( # noqa + CustomInventoryScript, + Group, + Host, + HostMetric, + Inventory, + InventorySource, + InventoryUpdate, + SmartInventoryMembership, +) from awx.main.models.jobs import ( # noqa Job, JobHostSummary, diff --git a/awx/main/models/events.py b/awx/main/models/events.py index 4f07bbb14e..25f8621672 100644 --- a/awx/main/models/events.py +++ b/awx/main/models/events.py @@ -510,12 +510,15 @@ class JobEvent(BasePlaybookEvent): job = self.job from awx.main.models import Host, JobHostSummary # circular import + from awx.main.models import Host, JobHostSummary, HostMetric all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name') existing_host_ids = set(h.id for h in all_hosts) summaries = dict() + updated_hosts_list = list() for host in hostnames: + updated_hosts_list.append(host) host_id = self.host_map.get(host, None) if host_id not in existing_host_ids: host_id = None @@ -546,6 +549,13 @@ class JobEvent(BasePlaybookEvent): Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100) + # bulk-create + current_time = now() + HostMetric.objects.bulk_create( + [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100 + ) + HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time) + @property def job_verbosity(self): return self.job.verbosity diff --git a/awx/main/models/inventory.py b/awx/main/models/inventory.py index 99dc7b837b..465277fe99 100644 --- a/awx/main/models/inventory.py +++ b/awx/main/models/inventory.py @@ -803,6 +803,12 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin): return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self.inventory) | Q(inventoryupdate__inventory_source__groups=self)) +class HostMetric(models.Model): + hostname = models.CharField(primary_key=True, max_length=512) + first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against')) + last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against')) + + class InventorySourceOptions(BaseModel): """ Common fields for InventorySource and InventoryUpdate. diff --git a/awx/main/tests/functional/models/test_host_metric.py b/awx/main/tests/functional/models/test_host_metric.py new file mode 100644 index 0000000000..1f560e474f --- /dev/null +++ b/awx/main/tests/functional/models/test_host_metric.py @@ -0,0 +1,22 @@ +import pytest +from django.utils.timezone import now +from awx.main.models import HostMetric + + +@pytest.mark.django_db +def test_host_metrics_generation(): + hostnames = [f'Host {i}' for i in range(100)] + current_time = now() + HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time) for h in hostnames]) + + # 3 assertions have to be made + # 1) if all the objects were created or not + assert HostMetric.objects.count() == len(hostnames) + + # 2) Match the hostnames stored in DB with the one passed in bulk_create + assert sorted([s.hostname for s in HostMetric.objects.all()]) == sorted(hostnames) + + # 3) Make sure that first_automation attribute is today's date + date_today = now().strftime('%Y-%m-%d') + result = HostMetric.objects.filter(first_automation__startswith=date_today).count() + assert result == len(hostnames)