Merge pull request #6588 from chrismeyersfsu/feature-fact_cache

initial tower fact cache implementation

awx/main/tests/functional/commands/test_run_fact_cache_receiver.py

@@ -1,129 +0,0 @@
-# Copyright (c) 2015 Ansible, Inc.
-# All Rights Reserved
-
-# Python
-import pytest
-from datetime import datetime
-import json
-
-# Django
-from django.utils import timezone
-
-# AWX
-from awx.main.management.commands.run_fact_cache_receiver import FactBrokerWorker
-from awx.main.models.fact import Fact
-from awx.main.models.inventory import Host
-from awx.main.models.base import PERM_INVENTORY_SCAN
-
-
-@pytest.fixture
-def mock_message(mocker):
-    class Message():
-        def ack():
-            pass
-    msg = Message()
-    mocker.patch.object(msg, 'ack')
-    return msg
-
-
-@pytest.fixture
-def mock_job_generator(mocker):
-    def fn(store_facts=True, job_type=PERM_INVENTORY_SCAN):
-        class Job():
-            def __init__(self):
-                self.store_facts = store_facts
-                self.job_type = job_type
-        job = Job()
-        mocker.patch('awx.main.models.Job.objects.get', return_value=job)
-        return job
-    return fn
-
-
-# TODO: Check that timestamp and other attributes are as expected
-def check_process_fact_message_module(fact_returned, data, module_name, message):
-    date_key = data['date_key']
-
-    message.ack.assert_called_with()
-
-    # Ensure 1, and only 1, fact created
-    timestamp = datetime.fromtimestamp(date_key, timezone.utc)
-    assert 1 == Fact.objects.all().count()
-
-    host_obj = Host.objects.get(name=data['host'], inventory__id=data['inventory_id'])
-    assert host_obj is not None
-    fact_known = Fact.get_host_fact(host_obj.id, module_name, timestamp)
-    assert fact_known is not None
-    assert fact_known == fact_returned
-
-    assert host_obj == fact_returned.host
-    if module_name == 'ansible':
-        assert data['facts'] == fact_returned.facts
-    else:
-        assert data['facts'][module_name] == fact_returned.facts
-    assert timestamp  == fact_returned.timestamp
-    assert module_name == fact_returned.module
-
-
-@pytest.mark.django_db
-def test_process_fact_message_ansible(fact_msg_ansible, monkeypatch_jsonbfield_get_db_prep_save, mock_message, mock_job_generator):
-    receiver = FactBrokerWorker(None)
-    mock_job_generator(store_facts=False, job_type=PERM_INVENTORY_SCAN)
-    fact_returned = receiver.process_fact_message(fact_msg_ansible, mock_message)
-    check_process_fact_message_module(fact_returned, fact_msg_ansible, 'ansible', mock_message)
-
-
-@pytest.mark.django_db
-def test_process_fact_message_packages(fact_msg_packages, monkeypatch_jsonbfield_get_db_prep_save, mock_message, mock_job_generator):
-    receiver = FactBrokerWorker(None)
-    mock_job_generator(store_facts=False, job_type=PERM_INVENTORY_SCAN)
-    fact_returned = receiver.process_fact_message(fact_msg_packages, mock_message)
-    check_process_fact_message_module(fact_returned, fact_msg_packages, 'packages', mock_message)
-
-
-@pytest.mark.django_db
-def test_process_fact_message_services(fact_msg_services, monkeypatch_jsonbfield_get_db_prep_save, mock_message, mock_job_generator):
-    receiver = FactBrokerWorker(None)
-    mock_job_generator(store_facts=False, job_type=PERM_INVENTORY_SCAN)
-    fact_returned = receiver.process_fact_message(fact_msg_services, mock_message)
-    check_process_fact_message_module(fact_returned, fact_msg_services, 'services', mock_message)
-
-
-@pytest.mark.django_db
-def test_process_facts_message_ansible_overwrite(fact_scans, fact_msg_ansible, monkeypatch_jsonbfield_get_db_prep_save, mock_message, mock_job_generator):
-    '''
-    We pickypack our fact sending onto the Ansible fact interface.
-    The interface is <hostname, facts>. Where facts is a json blob of all the facts.
-    This makes it hard to decipher what facts are new/changed.
-    Because of this, we handle the same fact module data being sent multiple times
-    and just keep the newest version.
-    '''
-    #epoch = timezone.now()
-    mock_job_generator(store_facts=False, job_type=PERM_INVENTORY_SCAN)
-    epoch = datetime.fromtimestamp(fact_msg_ansible['date_key'])
-    fact_scans(fact_scans=1, timestamp_epoch=epoch)
-    key = 'ansible.overwrite'
-    value = 'hello world'
-
-    receiver = FactBrokerWorker(None)
-    receiver.process_fact_message(fact_msg_ansible, mock_message)
-
-    fact_msg_ansible['facts'][key] = value
-    fact_returned = receiver.process_fact_message(fact_msg_ansible, mock_message)
-
-    fact_obj = Fact.objects.get(id=fact_returned.id)
-    assert key in fact_obj.facts
-    assert fact_msg_ansible['facts'] == (json.loads(fact_obj.facts) if isinstance(fact_obj.facts, unicode) else fact_obj.facts) # TODO: Just make response.data['facts'] when we're only dealing with postgres, or if jsonfields ever fixes this bug
-
-
-@pytest.mark.django_db
-def test_process_fact_store_facts(fact_msg_services, monkeypatch_jsonbfield_get_db_prep_save, mock_message, mock_job_generator):
-    receiver = FactBrokerWorker(None)
-    mock_job_generator(store_facts=True, job_type='run')
-    receiver.process_fact_message(fact_msg_services, mock_message)
-
-    host_obj = Host.objects.get(name=fact_msg_services['host'], inventory__id=fact_msg_services['inventory_id'])
-    assert host_obj is not None
-
-    assert host_obj.ansible_facts == fact_msg_services['facts']
-
-

awx/main/tests/unit/models/test_jobs.py

@@ -0,0 +1,133 @@
+import pytest
+
+from awx.main.models import (
+    Job,
+    Inventory,
+    Host,
+)
+
+import datetime
+import json
+from dateutil.tz import tzutc
+
+
+class CacheMock(object):
+    def __init__(self):
+        self.d = dict()
+
+    def get(self, key):
+        if key not in self.d:
+            return None
+        return self.d[key]
+
+    def set(self, key, val):
+        self.d[key] = val
+
+    def delete(self, key):
+        del self.d[key]
+
+
+@pytest.fixture
+def old_time():
+    return (datetime.datetime.now(tzutc()) - datetime.timedelta(minutes=60))
+
+
+@pytest.fixture()
+def new_time():
+    return (datetime.datetime.now(tzutc()))
+
+
+@pytest.fixture
+def hosts(old_time, inventory):
+    return [
+        Host(name='host1', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory),
+        Host(name='host2', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory),
+        Host(name='host3', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=old_time, inventory=inventory),
+    ]
+
+
+@pytest.fixture
+def hosts2(inventory):
+    return [
+        Host(name='host2', ansible_facts="foobar", ansible_facts_modified=old_time, inventory=inventory),
+    ]
+
+
+@pytest.fixture
+def inventory():
+    return Inventory(id=5)
+
+
+@pytest.fixture
+def mock_cache(mocker):
+    cache = CacheMock()
+    mocker.patch.object(cache, 'set', wraps=cache.set)
+    mocker.patch.object(cache, 'get', wraps=cache.get)
+    mocker.patch.object(cache, 'delete', wraps=cache.delete)
+    return cache
+
+
+@pytest.fixture
+def job(mocker, hosts, inventory, mock_cache):
+    j = Job(inventory=inventory, id=2)
+    j._get_inventory_hosts = mocker.Mock(return_value=hosts)
+    j._get_memcache_connection = mocker.Mock(return_value=mock_cache)
+    return j
+
+
+@pytest.fixture
+def job2(mocker, hosts2, inventory, mock_cache):
+    j = Job(inventory=inventory, id=3)
+    j._get_inventory_hosts = mocker.Mock(return_value=hosts2)
+    j._get_memcache_connection = mocker.Mock(return_value=mock_cache)
+    return j
+
+
+def test_start_job_fact_cache(hosts, job, inventory, mocker):
+
+    job.start_job_fact_cache()
+
+    job._get_memcache_connection().set.assert_any_call('5', [h.name for h in hosts])
+    for host in hosts:  
+        job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, host.name), json.dumps(host.ansible_facts))
+        job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, host.name), host.ansible_facts_modified.isoformat())
+
+
+def test_start_job_fact_cache_existing_host(hosts, hosts2, job, job2, inventory, mocker):
+
+    job.start_job_fact_cache()
+
+    for host in hosts:  
+        job._get_memcache_connection().set.assert_any_call('{}-{}'.format(5, host.name), json.dumps(host.ansible_facts))
+        job._get_memcache_connection().set.assert_any_call('{}-{}-modified'.format(5, host.name), host.ansible_facts_modified.isoformat())
+
+    job._get_memcache_connection().set.reset_mock()
+
+    job2.start_job_fact_cache()
+
+    # Ensure hosts2 ansible_facts didn't overwrite hosts ansible_facts
+    ansible_facts_cached = job._get_memcache_connection().get('{}-{}'.format(5, hosts2[0].name))
+    assert ansible_facts_cached == json.dumps(hosts[1].ansible_facts)
+
+
+def test_finish_job_fact_cache(job, hosts, inventory, mocker, new_time):
+
+    job.start_job_fact_cache()
+    for h in hosts:
+        h.save = mocker.Mock()
+
+    host_key = job.memcached_fact_host_key(hosts[1].name)
+    modified_key = job.memcached_fact_modified_key(hosts[1].name)
+
+    ansible_facts_new = {"foo": "bar", "insights": {"system_id": "updated_by_scan"}}
+    job._get_memcache_connection().set(host_key, json.dumps(ansible_facts_new))
+    job._get_memcache_connection().set(modified_key, new_time.isoformat())
+    
+    job.finish_job_fact_cache()
+
+    hosts[0].save.assert_not_called()
+    hosts[2].save.assert_not_called()
+    assert hosts[1].ansible_facts == ansible_facts_new
+    assert hosts[1].insights_system_id == "updated_by_scan"
+    hosts[1].save.assert_called_once_with()
+

awx/main/tests/unit/utils/test_handlers.py

@@ -4,6 +4,8 @@ import cStringIO
 import json
 import logging
 import socket
+import datetime
+from dateutil.tz import tzutc
 from uuid import uuid4
 
 import mock
@@ -135,7 +137,7 @@ def test_base_logging_handler_emit(dummy_log_record):
     assert body['message'] == 'User joe logged in'
 
 
-def test_base_logging_handler_emit_one_record_per_fact():
+def test_base_logging_handler_emit_system_tracking():
     handler = BaseHandler(host='127.0.0.1', enabled_flag=True,
                           message_type='logstash', indv_facts=True,
                           enabled_loggers=['awx', 'activity_stream', 'job_events', 'system_tracking'])
@@ -149,27 +151,20 @@ def test_base_logging_handler_emit_one_record_per_fact():
         tuple(), # args,
         None # exc_info
     )
-    record.module_name = 'packages'
-    record.facts_data = [{
-        "name": "ansible",
-        "version": "2.2.1.0"
-    }, {
-        "name": "ansible-tower",
-        "version": "3.1.0"
-    }]
+    record.inventory_id = 11
+    record.host_name = 'my_lucky_host'
+    record.ansible_facts = {
+        "ansible_kernel": "4.4.66-boot2docker",
+        "ansible_machine": "x86_64",
+        "ansible_swapfree_mb": 4663,
+    }
+    record.ansible_facts_modified = datetime.datetime.now(tzutc()).isoformat()
     sent_payloads = handler.emit(record)
 
-    assert len(sent_payloads) == 2
-    sent_payloads.sort(key=lambda payload: payload['version'])
-
+    assert len(sent_payloads) == 1
+    assert sent_payloads[0]['ansible_facts'] == record.ansible_facts
     assert sent_payloads[0]['level'] == 'INFO'
     assert sent_payloads[0]['logger_name'] == 'awx.analytics.system_tracking'
-    assert sent_payloads[0]['name'] == 'ansible'
-    assert sent_payloads[0]['version'] == '2.2.1.0'
-    assert sent_payloads[1]['level'] == 'INFO'
-    assert sent_payloads[1]['logger_name'] == 'awx.analytics.system_tracking'
-    assert sent_payloads[1]['name'] == 'ansible-tower'
-    assert sent_payloads[1]['version'] == '3.1.0'
 
 
 @pytest.mark.parametrize('host, port, normalized, hostname_only', [

awx/main/tests/unit/utils/test_reload.py

@@ -10,7 +10,7 @@ def test_produce_supervisor_command(mocker):
     with mocker.patch.object(reload.subprocess, 'Popen', Popen_mock):
         reload._supervisor_service_command(['beat', 'callback', 'fact'], "restart")
         reload.subprocess.Popen.assert_called_once_with(
-            ['supervisorctl', 'restart', 'tower-processes:receiver', 'tower-processes:factcacher'],
+            ['supervisorctl', 'restart', 'tower-processes:receiver',],
             stderr=-1, stdin=-1, stdout=-1)