add postgres Fact model, update views, tests

* awx.main.models Fact added
* view host fact and timeline updated to use new Postgres Fact model instead of Mongo
* Removed license set start Mongo logic
* added View tests
* added Model tests
* Removed mongo fact unit tests
* point at modified jsonbfield that supports sqlite storage driver
* postgresify fact cache receiver
* test OPTIONS endpoint
* Note: single fact view not implemented yet.
This commit is contained in:
Chris Meyers
2016-02-24 14:38:54 -05:00
parent 56b0da30f1
commit 7ffe46fc74
29 changed files with 5025 additions and 710 deletions
@@ -0,0 +1,61 @@
import mock
import pytest
from awx.api.views import (
ActivityStreamList,
ActivityStreamDetail,
OrganizationList,
)
from awx.main.middleware import ActivityStreamMiddleware
from awx.main.models.activity_stream import ActivityStream
from django.core.urlresolvers import reverse
def mock_feature_enabled(feature, bypass_database=None):
return True
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_get_activity_stream_list(monkeypatch, organization, get, user):
url = reverse('api:activity_stream_list')
response = get(ActivityStreamList, user('admin', True), url)
assert response.status_code == 200
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_basic_fields(monkeypatch, organization, get, user):
u = user('admin', True)
activity_stream = ActivityStream.objects.latest('pk')
activity_stream.actor = u
activity_stream.save()
aspk = activity_stream.pk
url = reverse('api:activity_stream_detail', args=(aspk,))
response = get(ActivityStreamDetail, user('admin', True), url, pk=aspk)
assert response.status_code == 200
assert 'related' in response.data
assert 'organization' in response.data['related']
assert 'summary_fields' in response.data
assert 'organization' in response.data['summary_fields']
assert response.data['summary_fields']['organization'][0]['name'] == 'test-org'
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_middleware_actor_added(monkeypatch, post, get, user):
u = user('admin-poster', True)
url = reverse('api:organization_list')
response = post(OrganizationList, u, url,
kwargs=dict(name='test-org', description='test-desc'),
middleware=ActivityStreamMiddleware())
assert response.status_code == 201
org_id = response.data['id']
activity_stream = ActivityStream.objects.filter(organization__pk=org_id).first()
url = reverse('api:activity_stream_detail', args=(activity_stream.pk,))
response = get(ActivityStreamDetail, u, url, pk=activity_stream.pk)
assert response.status_code == 200
assert response.data['summary_fields']['actor']['username'] == 'admin-poster'
@@ -0,0 +1,239 @@
# Python
import mock
import pytest
from datetime import timedelta
import urlparse
import urllib
# AWX
from awx.main.models.fact import Fact
from awx.api.views import (
HostFactVersionsList,
)
from awx.main.utils import timestamp_apiformat
# Django
from django.core.urlresolvers import reverse
from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
return True
def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), get_params={}, host_count=1):
hosts = hosts(host_count=host_count)
fact_scans(fact_scans=3, timestamp_epoch=epoch)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = get(HostFactVersionsList, user('admin', True), url, pk=hosts[0].id, params=get_params)
return (hosts[0], response)
def check_url(url1_full, fact_known, module):
url1_split = urlparse.urlsplit(url1_full)
url1 = url1_split.path
url1_params = urlparse.parse_qsl(url1_split.query)
url2 = reverse('api:host_fact_compare_view', args=(fact_known.host.pk,))
url2_params = [('module', module), ('datetime', timestamp_apiformat(fact_known.timestamp))]
assert url1 == url2
assert urllib.urlencode(url1_params) == urllib.urlencode(url2_params)
def check_response_facts(facts_known, response):
for i, fact_known in enumerate(facts_known):
assert fact_known.module == response.data['results'][i]['module']
assert timestamp_apiformat(fact_known.timestamp) == response.data['results'][i]['timestamp']
check_url(response.data['results'][i]['related']['fact_view'], fact_known, fact_known.module)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_no_facts_db(hosts, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = get(HostFactVersionsList, user('admin', True), url, pk=hosts[0].id)
response_expected = {
'results': []
}
assert response_expected == response.data
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_basic_fields(hosts, fact_scans, get, user):
epoch = timezone.now()
search = {
'from': epoch,
'to': epoch,
}
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, get_params=search)
results = response.data['results']
assert 'related' in results[0]
assert 'timestamp' in results[0]
assert 'module' in results[0]
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
@pytest.mark.skipif(True, reason="Options fix landed in devel but not here. Enable this after this pr gets merged.")
def test_basic_options_fields(hosts, fact_scans, options, user):
hosts = hosts(host_count=1)
fact_scans(fact_scans=1)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = options(HostFactVersionsList, user('admin', True), url, pk=hosts[0].id)
#import json
#print(json.dumps(response.data))
assert 'related' in response.data
assert 'id' in response.data
assert 'facts' in response.data
assert 'module' in response.data
assert 'host' in response.data
assert isinstance(response.data['host'], int)
assert 'summary_fields' in response.data
assert 'host' in response.data['summary_fields']
assert 'name' in response.data['summary_fields']['host']
assert 'description' in response.data['summary_fields']['host']
assert 'host' in response.data['related']
assert reverse('api:host_detail', args=(hosts[0].pk,)) == response.data['related']['host']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_related_fact_view(hosts, fact_scans, get, user):
epoch = timezone.now()
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch)
facts_known = Fact.get_timeline(host.id)
assert 9 == len(facts_known)
assert 9 == len(response.data['results'])
for i, fact_known in enumerate(facts_known):
check_url(response.data['results'][i]['related']['fact_view'], fact_known, fact_known.module)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_multiple_hosts(hosts, fact_scans, get, user):
epoch = timezone.now()
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, host_count=3)
facts_known = Fact.get_timeline(host.id)
assert 9 == len(facts_known)
assert 9 == len(response.data['results'])
for i, fact_known in enumerate(facts_known):
check_url(response.data['results'][i]['related']['fact_view'], fact_known, fact_known.module)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_param_to_from(hosts, fact_scans, get, user):
epoch = timezone.now()
search = {
'from': epoch - timedelta(days=10),
'to': epoch + timedelta(days=10),
}
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, get_params=search)
facts_known = Fact.get_timeline(host.id, ts_from=search['from'], ts_to=search['to'])
assert 9 == len(facts_known)
assert 9 == len(response.data['results'])
check_response_facts(facts_known, response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_param_module(hosts, fact_scans, get, user):
epoch = timezone.now()
search = {
'module': 'packages',
}
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, get_params=search)
facts_known = Fact.get_timeline(host.id, module=search['module'])
assert 3 == len(facts_known)
assert 3 == len(response.data['results'])
check_response_facts(facts_known, response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_param_from(hosts, fact_scans, get, user):
epoch = timezone.now()
search = {
'from': epoch + timedelta(days=1),
}
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, get_params=search)
facts_known = Fact.get_timeline(host.id, ts_from=search['from'])
assert 3 == len(facts_known)
assert 3 == len(response.data['results'])
check_response_facts(facts_known, response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_param_to(hosts, fact_scans, get, user):
epoch = timezone.now()
search = {
'to': epoch + timedelta(days=1),
}
(host, response) = setup_common(hosts, fact_scans, get, user, epoch=epoch, get_params=search)
facts_known = Fact.get_timeline(host.id, ts_to=search['to'])
assert 6 == len(facts_known)
assert 6 == len(response.data['results'])
check_response_facts(facts_known, response)
def _test_user_access_control(hosts, fact_scans, get, user_obj, team_obj):
hosts = hosts(host_count=1)
fact_scans(fact_scans=1)
team_obj.users.add(user_obj)
url = reverse('api:host_fact_versions_list', args=(hosts[0].pk,))
response = get(HostFactVersionsList, user_obj, url, pk=hosts[0].id)
return response
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_normal_user_403(hosts, fact_scans, get, user, team):
user_bob = user('bob', False)
response = _test_user_access_control(hosts, fact_scans, get, user_bob, team)
assert 403 == response.status_code
assert "You do not have permission to perform this action." == response.data['detail']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_super_user_ok(hosts, fact_scans, get, user, team):
user_super = user('bob', True)
response = _test_user_access_control(hosts, fact_scans, get, user_super, team)
assert 200 == response.status_code
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_user_admin_ok(organization, hosts, fact_scans, get, user, team):
user_admin = user('johnson', False)
organization.admins.add(user_admin)
response = _test_user_access_control(hosts, fact_scans, get, user_admin, team)
assert 200 == response.status_code
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_user_admin_403(organization, organizations, hosts, fact_scans, get, user, team):
user_admin = user('johnson', False)
org2 = organizations(1)
org2[0].admins.add(user_admin)
response = _test_user_access_control(hosts, fact_scans, get, user_admin, team)
assert 403 == response.status_code
@@ -0,0 +1,158 @@
import mock
import pytest
import json
from awx.api.views import (
HostFactCompareView,
)
from awx.main.utils import timestamp_apiformat
from django.core.urlresolvers import reverse
from django.utils import timezone
def mock_feature_enabled(feature, bypass_database=None):
return True
# TODO: Consider making the fact_scan() fixture a Class, instead of a function, and move this method into it
def find_fact(facts, host_id, module_name, timestamp):
for f in facts:
if f.host_id == host_id and f.module == module_name and f.timestamp == timestamp:
return f
raise RuntimeError('fact <%s, %s, %s> not found in %s', (host_id, module_name, timestamp, facts))
def setup_common(hosts, fact_scans, get, user, epoch=timezone.now(), module_name='ansible', get_params={}):
hosts = hosts(host_count=1)
facts = fact_scans(fact_scans=1, timestamp_epoch=epoch)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = get(HostFactCompareView, user('admin', True), url, pk=hosts[0].id, params=get_params)
fact_known = find_fact(facts, hosts[0].id, module_name, epoch)
return (fact_known, response)
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_no_fact_found(hosts, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = get(HostFactCompareView, user('admin', True), url, pk=hosts[0].id)
expected_response = {
"detail": "Fact not found"
}
assert 404 == response.status_code
assert expected_response == response.data
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_basic_fields(hosts, fact_scans, get, user):
hosts = hosts(host_count=1)
fact_scans(fact_scans=1)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = get(HostFactCompareView, user('admin', True), url, pk=hosts[0].id)
assert 'related' in response.data
assert 'id' in response.data
assert 'facts' in response.data
assert 'module' in response.data
assert 'host' in response.data
assert isinstance(response.data['host'], int)
assert 'summary_fields' in response.data
assert 'host' in response.data['summary_fields']
assert 'name' in response.data['summary_fields']['host']
assert 'description' in response.data['summary_fields']['host']
assert 'host' in response.data['related']
assert reverse('api:host_detail', args=(hosts[0].pk,)) == response.data['related']['host']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_content(hosts, fact_scans, get, user, fact_ansible_json):
(fact_known, response) = setup_common(hosts, fact_scans, get, user)
assert fact_known.host_id == response.data['host']
assert fact_ansible_json == json.loads(response.data['facts'])
assert timestamp_apiformat(fact_known.timestamp) == response.data['timestamp']
assert fact_known.module == response.data['module']
def _test_search_by_module(hosts, fact_scans, get, user, fact_json, module_name):
params = {
'module': module_name
}
(fact_known, response) = setup_common(hosts, fact_scans, get, user, module_name=module_name, get_params=params)
assert fact_json == json.loads(response.data['facts'])
assert timestamp_apiformat(fact_known.timestamp) == response.data['timestamp']
assert module_name == response.data['module']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_search_by_module_packages(hosts, fact_scans, get, user, fact_packages_json):
_test_search_by_module(hosts, fact_scans, get, user, fact_packages_json, 'packages')
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_search_by_module_services(hosts, fact_scans, get, user, fact_services_json):
_test_search_by_module(hosts, fact_scans, get, user, fact_services_json, 'services')
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.django_db
def test_search_by_timestamp_and_module(hosts, fact_scans, get, user, fact_packages_json):
epoch = timezone.now()
module_name = 'packages'
(fact_known, response) = setup_common(hosts, fact_scans, get, user, module_name=module_name, epoch=epoch, get_params=dict(module=module_name, datetime=epoch))
assert fact_known.id == response.data['id']
def _test_user_access_control(hosts, fact_scans, get, user_obj, team_obj):
hosts = hosts(host_count=1)
fact_scans(fact_scans=1)
team_obj.users.add(user_obj)
url = reverse('api:host_fact_compare_view', args=(hosts[0].pk,))
response = get(HostFactCompareView, user_obj, url, pk=hosts[0].id)
return response
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_normal_user_403(hosts, fact_scans, get, user, team):
user_bob = user('bob', False)
response = _test_user_access_control(hosts, fact_scans, get, user_bob, team)
assert 403 == response.status_code
assert "You do not have permission to perform this action." == response.data['detail']
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_super_user_ok(hosts, fact_scans, get, user, team):
user_super = user('bob', True)
response = _test_user_access_control(hosts, fact_scans, get, user_super, team)
assert 200 == response.status_code
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_user_admin_ok(organization, hosts, fact_scans, get, user, team):
user_admin = user('johnson', False)
organization.admins.add(user_admin)
response = _test_user_access_control(hosts, fact_scans, get, user_admin, team)
assert 200 == response.status_code
@mock.patch('awx.api.views.feature_enabled', new=mock_feature_enabled)
@pytest.mark.ac
@pytest.mark.django_db
def test_user_admin_403(organization, organizations, hosts, fact_scans, get, user, team):
user_admin = user('johnson', False)
org2 = organizations(1)
org2[0].admins.add(user_admin)
response = _test_user_access_control(hosts, fact_scans, get, user_admin, team)
assert 403 == response.status_code
@@ -0,0 +1,20 @@
# TODO: As of writing this our only concern is ensuring that the fact feature is reflected in the Host endpoint.
# Other host tests should live here to make this test suite more complete.
import pytest
from awx.api.views import (
HostDetail,
)
from django.core.urlresolvers import reverse
@pytest.mark.django_db
def test_basic_fields(hosts, fact_scans, get, user):
hosts = hosts(host_count=1)
url = reverse('api:host_detail', args=(hosts[0].pk,))
response = get(HostDetail, user('admin', True), url, pk=hosts[0].id)
assert 'related' in response.data
assert 'fact_versions' in response.data['related']
assert reverse('api:host_fact_versions_list', args=(hosts[0].pk,)) == response.data['related']['fact_versions']