Merge pull request #9654 from amolgautam25/issue_4687

Adding host_metrics for customers who don't have automation analytics

SUMMARY

I am trying to accomplish the following:

 Create a new model that stores details about hostname
 create corresponding migrations for it
 Update the table (main_hostmetrics) after playbook execution
 retrieve the data with custom awx-manage command :  awx-manage host_metrics --since <date> --until <date>


ISSUE TYPE


Feature Pull Request

COMPONENT NAME


API

AWX VERSION

awx: 18.0.0


ADDITIONAL INFORMATION

Reviewed-by: Ryan Petrello <ryan@ryanpetrello.com>
Reviewed-by: Amol Gautam <amol_gautam25@yahoo.co.in>
Reviewed-by: Chris Meyers <None>
Reviewed-by: Jeff Bradberry <None>
Reviewed-by: Bill Nottingham <None>
Reviewed-by: Ladislav Smola <lsmola@redhat.com>
Reviewed-by: Alan Rominger <arominge@redhat.com>

awx/main/management/commands/host_metric.py

@@ -0,0 +1,54 @@
+from django.core.management.base import BaseCommand
+import datetime
+from django.core.serializers.json import DjangoJSONEncoder
+from awx.main.models.inventory import HostMetric
+import json
+
+
+class Command(BaseCommand):
+
+    help = 'This is for offline licensing usage'
+
+    def add_arguments(self, parser):
+        parser.add_argument('--since', type=datetime.datetime.fromisoformat, help='Start Date in ISO format')
+        parser.add_argument('--until', type=datetime.datetime.fromisoformat, help='End Date in ISO format')
+        parser.add_argument('--json', action='store_true', help='Select output as JSON')
+
+    def handle(self, *args, **options):
+        since = options.get('since')
+        until = options.get('until')
+
+        if since is None and until is None:
+            print("No Arguments received")
+            return None
+
+        if since is not None and since.tzinfo is None:
+            since = since.replace(tzinfo=datetime.timezone.utc)
+
+        if until is not None and until.tzinfo is None:
+            until = until.replace(tzinfo=datetime.timezone.utc)
+
+        filter_kwargs = {}
+        if since is not None:
+            filter_kwargs['last_automation__gte'] = since
+        if until is not None:
+            filter_kwargs['last_automation__lte'] = until
+
+        result = HostMetric.objects.filter(**filter_kwargs)
+
+        # if --json flag is set, output the result in json format
+        if options['json']:
+            list_of_queryset = list(result.values('hostname', 'first_automation', 'last_automation'))
+            json_result = json.dumps(list_of_queryset, cls=DjangoJSONEncoder)
+            print(json_result)
+
+        # --json flag is not set, output in plain text
+        else:
+            print(f"Total Number of hosts automated: {len(result)}")
+            for item in result:
+                print(
+                    "Hostname : {hostname} | first_automation : {first_automation} | last_automation : {last_automation}".format(
+                        hostname=item.hostname, first_automation=item.first_automation, last_automation=item.last_automation
+                    )
+                )
+        return

awx/main/migrations/0143_hostmetric.py

@@ -0,0 +1,21 @@
+# Generated by Django 2.2.16 on 2021-05-18 18:08
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('main', '0142_update_ee_image_field_description'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='HostMetric',
+            fields=[
+                ('hostname', models.CharField(max_length=512, primary_key=True, serialize=False)),
+                ('first_automation', models.DateTimeField(auto_now_add=True, db_index=True, help_text='When the host was first automated against')),
+                ('last_automation', models.DateTimeField(db_index=True, help_text='When the host was last automated against')),
+            ],
+        ),
+    ]

awx/main/models/__init__.py

@@ -12,7 +12,16 @@ from awx.main.models.unified_jobs import UnifiedJob, UnifiedJobTemplate, StdoutM
 from awx.main.models.organization import Organization, Profile, Team, UserSessionMembership  # noqa
 from awx.main.models.credential import Credential, CredentialType, CredentialInputSource, ManagedCredentialType, build_safe_env  # noqa
 from awx.main.models.projects import Project, ProjectUpdate  # noqa
-from awx.main.models.inventory import Group, Host, Inventory, InventorySource, InventoryUpdate, SmartInventoryMembership  # noqa
+from awx.main.models.inventory import (  # noqa
+    CustomInventoryScript,
+    Group,
+    Host,
+    HostMetric,
+    Inventory,
+    InventorySource,
+    InventoryUpdate,
+    SmartInventoryMembership,
+)
 from awx.main.models.jobs import (  # noqa
     Job,
     JobHostSummary,

awx/main/models/events.py

@@ -510,12 +510,15 @@ class JobEvent(BasePlaybookEvent):
             job = self.job
 
             from awx.main.models import Host, JobHostSummary  # circular import
+            from awx.main.models import Host, JobHostSummary, HostMetric
 
             all_hosts = Host.objects.filter(pk__in=self.host_map.values()).only('id', 'name')
             existing_host_ids = set(h.id for h in all_hosts)
 
             summaries = dict()
+            updated_hosts_list = list()
             for host in hostnames:
+                updated_hosts_list.append(host)
                 host_id = self.host_map.get(host, None)
                 if host_id not in existing_host_ids:
                     host_id = None
@@ -546,6 +549,13 @@ class JobEvent(BasePlaybookEvent):
 
             Host.objects.bulk_update(list(updated_hosts), ['last_job_id', 'last_job_host_summary_id'], batch_size=100)
 
+            # bulk-create
+            current_time = now()
+            HostMetric.objects.bulk_create(
+                [HostMetric(hostname=hostname, last_automation=current_time) for hostname in updated_hosts_list], ignore_conflicts=True, batch_size=100
+            )
+            HostMetric.objects.filter(hostname__in=updated_hosts_list).update(last_automation=current_time)
+
     @property
     def job_verbosity(self):
         return self.job.verbosity

awx/main/models/inventory.py

@@ -804,6 +804,12 @@ class Group(CommonModelNameNotUnique, RelatedJobsMixin):
         return UnifiedJob.objects.non_polymorphic().filter(Q(job__inventory=self.inventory) | Q(inventoryupdate__inventory_source__groups=self))
 
 
+class HostMetric(models.Model):
+    hostname = models.CharField(primary_key=True, max_length=512)
+    first_automation = models.DateTimeField(auto_now_add=True, null=False, db_index=True, help_text=_('When the host was first automated against'))
+    last_automation = models.DateTimeField(db_index=True, help_text=_('When the host was last automated against'))
+
+
 class InventorySourceOptions(BaseModel):
     """
     Common fields for InventorySource and InventoryUpdate.

awx/main/tests/functional/models/test_host_metric.py

@@ -0,0 +1,22 @@
+import pytest
+from django.utils.timezone import now
+from awx.main.models import HostMetric
+
+
+@pytest.mark.django_db
+def test_host_metrics_generation():
+    hostnames = [f'Host {i}' for i in range(100)]
+    current_time = now()
+    HostMetric.objects.bulk_create([HostMetric(hostname=h, last_automation=current_time) for h in hostnames])
+
+    # 3 assertions have to be made
+    # 1) if all the objects were created or not
+    assert HostMetric.objects.count() == len(hostnames)
+
+    # 2) Match the hostnames stored in DB with the one passed in bulk_create
+    assert sorted([s.hostname for s in HostMetric.objects.all()]) == sorted(hostnames)
+
+    # 3) Make sure that first_automation attribute is today's date
+    date_today = now().strftime('%Y-%m-%d')
+    result = HostMetric.objects.filter(first_automation__startswith=date_today).count()
+    assert result == len(hostnames)