Merge pull request #8109 from AlanCoding/hack_null_org

Hack to delete orphaned organizations, consolidate get_one methods

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul
This commit is contained in:
softwarefactory-project-zuul[bot] 2020-09-10 18:15:45 +00:00 committed by GitHub
commit 4b51c71220
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
16 changed files with 166 additions and 120 deletions

View File

@ -4,11 +4,9 @@ __metaclass__ = type
from . tower_module import TowerModule
from ansible.module_utils.urls import Request, SSLValidationError, ConnectionError
from ansible.module_utils.six import PY2
from ansible.module_utils.six.moves.urllib.parse import urlencode
from ansible.module_utils.six.moves.urllib.error import HTTPError
from ansible.module_utils.six.moves.http_cookiejar import CookieJar
import time
import re
from json import loads, dumps
@ -117,22 +115,23 @@ class TowerAPIModule(TowerModule):
response['json']['next'] = next_page
return response
def get_one(self, endpoint, name_or_id=None, *args, **kwargs):
def get_one(self, endpoint, name_or_id=None, allow_none=True, **kwargs):
new_kwargs = kwargs.copy()
if name_or_id:
name_field = self.get_name_field_from_endpoint(endpoint)
new_args = kwargs.get('data', {}).copy()
if name_field in new_args:
new_data = kwargs.get('data', {}).copy()
if name_field in new_data:
self.fail_json(msg="You can't specify the field {0} in your search data if using the name_or_id field".format(name_field))
new_args['or__{0}'.format(name_field)] = name_or_id
try:
new_args['or__id'] = int(name_or_id)
new_data['or__id'] = int(name_or_id)
new_data['or__{0}'.format(name_field)] = name_or_id
except ValueError:
# If we get a value error, then we didn't have an integer so we can just pass and fall down to the fail
pass
kwargs['data'] = new_args
new_data[name_field] = name_or_id
new_kwargs['data'] = new_data
response = self.get_endpoint(endpoint, *args, **kwargs)
response = self.get_endpoint(endpoint, **new_kwargs)
if response['status_code'] != 200:
fail_msg = "Got a {0} response when trying to get one from {1}".format(response['status_code'], endpoint)
if 'detail' in response.get('json', {}):
@ -143,63 +142,52 @@ class TowerAPIModule(TowerModule):
self.fail_json(msg="The endpoint did not provide count and results")
if response['json']['count'] == 0:
return None
if allow_none:
return None
else:
self.fail_wanted_one(response, endpoint, new_kwargs.get('data'))
elif response['json']['count'] > 1:
if name_or_id:
# Since we did a name or ID search and got > 1 return something if the id matches
for asset in response['json']['results']:
if asset['id'] == name_or_id:
if str(asset['id']) == name_or_id:
return asset
# We got > 1 and either didn't find something by ID (which means multiple names)
# Or we weren't running with a or search and just got back too many to begin with.
self.fail_json(msg="An unexpected number of items was returned from the API ({0})".format(response['json']['count']))
self.fail_wanted_one(response, endpoint, new_kwargs.get('data'))
return response['json']['results'][0]
def get_one_by_name_or_id(self, endpoint, name_or_id):
name_field = self.get_name_field_from_endpoint(endpoint)
def fail_wanted_one(self, response, endpoint, query_params):
sample = response.copy()
if len(sample['json']['results']) > 1:
sample['json']['results'] = sample['json']['results'][:2] + ['...more results snipped...']
url = self.build_url(endpoint, query_params)
display_endpoint = url.geturl()[len(self.host):] # truncate to not include the base URL
self.fail_json(
msg="Request to {0} returned {1} items, expected 1".format(
display_endpoint, response['json']['count']
),
query=query_params,
response=sample,
total_results=response['json']['count']
)
query_params = {'or__{0}'.format(name_field): name_or_id}
try:
query_params['or__id'] = int(name_or_id)
except ValueError:
# If we get a value error, then we didn't have an integer so we can just pass and fall down to the fail
pass
response = self.get_endpoint(endpoint, **{'data': query_params})
if response['status_code'] != 200:
self.fail_json(
msg="Failed to query endpoint {0} for {1} {2} ({3}), see results".format(endpoint, name_field, name_or_id, response['status_code']),
resuls=response
)
if response['json']['count'] == 1:
return response['json']['results'][0]
elif response['json']['count'] > 1:
for tower_object in response['json']['results']:
# ID takes priority, so we match on that first
if str(tower_object['id']) == name_or_id:
return tower_object
# We didn't match on an ID but we found more than 1 object, therefore the results are ambiguous
self.fail_json(msg="The requested name or id was ambiguous and resulted in too many items")
elif response['json']['count'] == 0:
self.fail_json(msg="The {0} {1} was not found on the Tower server".format(endpoint, name_or_id))
def get_exactly_one(self, endpoint, name_or_id=None, **kwargs):
return self.get_one(endpoint, name_or_id=name_or_id, allow_none=False, **kwargs)
def resolve_name_to_id(self, endpoint, name_or_id):
return self.get_one_by_name_or_id(endpoint, name_or_id)['id']
return self.get_exactly_one(endpoint, name_or_id)['id']
def make_request(self, method, endpoint, *args, **kwargs):
# In case someone is calling us directly; make sure we were given a method, let's not just assume a GET
if not method:
raise Exception("The HTTP method must be defined")
# Make sure we start with /api/vX
if not endpoint.startswith("/"):
endpoint = "/{0}".format(endpoint)
if not endpoint.startswith("/api/"):
endpoint = "/api/v2{0}".format(endpoint)
if not endpoint.endswith('/') and '?' not in endpoint:
endpoint = "{0}/".format(endpoint)
if method in ['POST', 'PUT', 'PATCH']:
url = self.build_url(endpoint)
else:
url = self.build_url(endpoint, query_params=kwargs.get('data'))
# Extract the headers, this will be used in a couple of places
headers = kwargs.get('headers', {})
@ -212,46 +200,41 @@ class TowerAPIModule(TowerModule):
# If we have a oauth token, we just use a bearer header
headers['Authorization'] = 'Bearer {0}'.format(self.oauth_token)
# Update the URL path with the endpoint
self.url = self.url._replace(path=endpoint)
if method in ['POST', 'PUT', 'PATCH']:
headers.setdefault('Content-Type', 'application/json')
kwargs['headers'] = headers
elif kwargs.get('data'):
self.url = self.url._replace(query=urlencode(kwargs.get('data')))
data = None # Important, if content type is not JSON, this should not be dict type
if headers.get('Content-Type', '') == 'application/json':
data = dumps(kwargs.get('data', {}))
try:
response = self.session.open(method, self.url.geturl(), headers=headers, validate_certs=self.verify_ssl, follow_redirects=True, data=data)
response = self.session.open(method, url.geturl(), headers=headers, validate_certs=self.verify_ssl, follow_redirects=True, data=data)
except(SSLValidationError) as ssl_err:
self.fail_json(msg="Could not establish a secure connection to your host ({1}): {0}.".format(self.url.netloc, ssl_err))
self.fail_json(msg="Could not establish a secure connection to your host ({1}): {0}.".format(url.netloc, ssl_err))
except(ConnectionError) as con_err:
self.fail_json(msg="There was a network error of some kind trying to connect to your host ({1}): {0}.".format(self.url.netloc, con_err))
self.fail_json(msg="There was a network error of some kind trying to connect to your host ({1}): {0}.".format(url.netloc, con_err))
except(HTTPError) as he:
# Sanity check: Did the server send back some kind of internal error?
if he.code >= 500:
self.fail_json(msg='The host sent back a server error ({1}): {0}. Please check the logs and try again later'.format(self.url.path, he))
self.fail_json(msg='The host sent back a server error ({1}): {0}. Please check the logs and try again later'.format(url.path, he))
# Sanity check: Did we fail to authenticate properly? If so, fail out now; this is always a failure.
elif he.code == 401:
self.fail_json(msg='Invalid Tower authentication credentials for {0} (HTTP 401).'.format(self.url.path))
self.fail_json(msg='Invalid Tower authentication credentials for {0} (HTTP 401).'.format(url.path))
# Sanity check: Did we get a forbidden response, which means that the user isn't allowed to do this? Report that.
elif he.code == 403:
self.fail_json(msg="You don't have permission to {1} to {0} (HTTP 403).".format(self.url.path, method))
self.fail_json(msg="You don't have permission to {1} to {0} (HTTP 403).".format(url.path, method))
# Sanity check: Did we get a 404 response?
# Requests with primary keys will return a 404 if there is no response, and we want to consistently trap these.
elif he.code == 404:
if kwargs.get('return_none_on_404', False):
return None
self.fail_json(msg='The requested object could not be found at {0}.'.format(self.url.path))
self.fail_json(msg='The requested object could not be found at {0}.'.format(url.path))
# Sanity check: Did we get a 405 response?
# A 405 means we used a method that isn't allowed. Usually this is a bad request, but it requires special treatment because the
# API sends it as a logic error in a few situations (e.g. trying to cancel a job that isn't running).
elif he.code == 405:
self.fail_json(msg="The Tower server says you can't make a request with the {0} method to this endpoing {1}".format(method, self.url.path))
self.fail_json(msg="The Tower server says you can't make a request with the {0} method to this endpoing {1}".format(method, url.path))
# Sanity check: Did we get some other kind of error? If so, write an appropriate error message.
elif he.code >= 400:
# We are going to return a 400 so the module can decide what to do with it
@ -265,11 +248,9 @@ class TowerAPIModule(TowerModule):
# A 204 is a normal response for a delete function
pass
else:
self.fail_json(msg="Unexpected return code when calling {0}: {1}".format(self.url.geturl(), he))
self.fail_json(msg="Unexpected return code when calling {0}: {1}".format(url.geturl(), he))
except(Exception) as e:
self.fail_json(msg="There was an unknown error when trying to connect to {2}: {0} {1}".format(type(e).__name__, e, self.url.geturl()))
finally:
self.url = self.url._replace(query=None)
self.fail_json(msg="There was an unknown error when trying to connect to {2}: {0} {1}".format(type(e).__name__, e, url.geturl()))
if not self.version_checked:
# In PY2 we get back an HTTPResponse object but PY2 is returning an addinfourl
@ -627,7 +608,7 @@ class TowerAPIModule(TowerModule):
# If we are past our time out fail with a message
if timeout and timeout < time.time() - start:
# Account for Legacy messages
if object_type is 'legacy_job_wait':
if object_type == 'legacy_job_wait':
self.json_output['msg'] = 'Monitoring of Job - {0} aborted due to timeout'.format(object_name)
else:
self.json_output['msg'] = 'Monitoring of {0} - {1} aborted due to timeout'.format(object_type, object_name)
@ -643,7 +624,7 @@ class TowerAPIModule(TowerModule):
# If the job has failed, we want to raise a task failure for that so we get a non-zero response.
if result['json']['failed']:
# Account for Legacy messages
if object_type is 'legacy_job_wait':
if object_type == 'legacy_job_wait':
self.json_output['msg'] = 'Job with id {0} failed'.format(object_name)
else:
self.json_output['msg'] = 'The {0} - {1}, failed'.format(object_type, object_name)

View File

@ -2,9 +2,9 @@ from __future__ import absolute_import, division, print_function
__metaclass__ = type
from ansible.module_utils.basic import AnsibleModule, env_fallback
from ansible.module_utils.six import PY2, string_types
from ansible.module_utils.six import string_types
from ansible.module_utils.six.moves import StringIO
from ansible.module_utils.six.moves.urllib.parse import urlparse
from ansible.module_utils.six.moves.urllib.parse import urlparse, urlencode
from ansible.module_utils.six.moves.configparser import ConfigParser, NoOptionError
from socket import gethostbyname
import re
@ -112,6 +112,23 @@ class TowerModule(AnsibleModule):
except Exception as e:
self.fail_json(msg="Unable to resolve tower_host ({1}): {0}".format(hostname, e))
def build_url(self, endpoint, query_params=None):
# Make sure we start with /api/vX
if not endpoint.startswith("/"):
endpoint = "/{0}".format(endpoint)
if not endpoint.startswith("/api/"):
endpoint = "/api/v2{0}".format(endpoint)
if not endpoint.endswith('/') and '?' not in endpoint:
endpoint = "{0}/".format(endpoint)
# Update the URL path with the endpoint
url = self.url._replace(path=endpoint)
if query_params:
url = url._replace(query=urlencode(query_params))
return url
def load_config_files(self):
# Load configs like TowerCLI would have from least import to most
config_files = ['/etc/tower/tower_cli.cfg', join(expanduser("~"), ".{0}".format(self.config_name))]

View File

@ -104,7 +104,6 @@ options:
description:
- Name of organization for project.
type: str
required: True
state:
description:
- Desired state of the resource.
@ -200,7 +199,7 @@ def main():
allow_override=dict(type='bool', aliases=['scm_allow_override']),
timeout=dict(type='int', default=0, aliases=['job_timeout']),
custom_virtualenv=dict(),
organization=dict(required=True),
organization=dict(),
notification_templates_started=dict(type="list", elements='str'),
notification_templates_success=dict(type="list", elements='str'),
notification_templates_error=dict(type="list", elements='str'),
@ -234,21 +233,22 @@ def main():
wait = module.params.get('wait')
# Attempt to look up the related items the user specified (these will fail the module if not found)
org_id = module.resolve_name_to_id('organizations', organization)
if credential is not None:
credential = module.resolve_name_to_id('credentials', credential)
lookup_data = {}
org_id = None
if organization:
org_id = module.resolve_name_to_id('organizations', organization)
lookup_data['organization'] = org_id
# Attempt to look up project based on the provided name and org ID
project = module.get_one('projects', name_or_id=name, **{
'data': {
'organization': org_id
}
})
project = module.get_one('projects', name_or_id=name, data=lookup_data)
if state == 'absent':
# If the state was absent we can let the module delete it if needed, the module will handle exiting from this
module.delete_if_needed(project)
if credential is not None:
credential = module.resolve_name_to_id('credentials', credential)
# Attempt to look up associated field items the user specified.
association_fields = {}

View File

@ -129,12 +129,7 @@ def main():
resource_name = params.get(param)
if resource_name:
resource = module.get_one_by_name_or_id(module.param_to_endpoint(param), resource_name)
if not resource:
module.fail_json(
msg='Failed to update role, {0} not found in {1}'.format(param, endpoint),
changed=False
)
resource = module.get_exactly_one(module.param_to_endpoint(param), resource_name)
resource_data[param] = resource
# separate actors from resources

View File

@ -127,7 +127,9 @@ def test_missing_credential_type(run_module, admin_user, organization):
state='present'
), admin_user)
assert result.get('failed', False), result
assert 'foobar was not found on the Tower server' in result['msg']
assert 'credential_type' in result['msg']
assert 'foobar' in result['msg']
assert 'returned 0 items, expected 1' in result['msg']
@pytest.mark.django_db

View File

@ -4,7 +4,7 @@ __metaclass__ = type
import json
import sys
from awx.main.models import Organization, Team
from awx.main.models import Organization, Team, Project, Inventory
from requests.models import Response
from unittest import mock
@ -125,3 +125,19 @@ def test_conflicting_name_and_id(run_module, admin_user):
'Lookup by id should be preferenced over name in cases of conflict.'
)
assert team.organization.name == 'foo'
def test_multiple_lookup(run_module, admin_user):
org1 = Organization.objects.create(name='foo')
org2 = Organization.objects.create(name='bar')
inv = Inventory.objects.create(name='Foo Inv')
proj1 = Project.objects.create(name='foo', organization=org1, scm_type='git', scm_url="https://github.com/ansible/ansible-tower-samples",)
proj2 = Project.objects.create(name='foo', organization=org2, scm_type='git', scm_url="https://github.com/ansible/ansible-tower-samples",)
result = run_module('tower_job_template', {
'name': 'Demo Job Template', 'project': proj1.name, 'inventory': inv.id, 'playbook': 'hello_world.yml'
}, admin_user)
assert result.get('failed', False)
assert 'projects' in result['msg']
assert 'foo' in result['msg']
assert 'returned 2 items, expected 1' in result['msg']
assert 'query' in result

View File

@ -3,6 +3,18 @@
tower_organization:
name: Default
- name: HACK - delete orphaned projects from preload data where organization deletd
tower_project:
name: "{{ item['id'] }}"
scm_type: git
state: absent
loop: >
{{ query('awx.awx.tower_api', 'projects',
query_params={'organization__isnull': true, 'name': 'Demo Project'})
}}
loop_control:
label: "Deleting Demo Project with null organization id={{ item['id'] }}"
- name: Assure that demo project exists
tower_project:
name: "Demo Project"

View File

@ -288,7 +288,7 @@
- name: Create an invalid SSH credential (Organization not found)
tower_credential:
name: SSH Credential
organization: Missing Organization
organization: Missing_Organization
state: present
kind: ssh
username: joe
@ -298,7 +298,9 @@
- assert:
that:
- "result is failed"
- "'The organizations Missing Organization was not found on the Tower server' in result.msg"
- "result is not changed"
- "'Missing_Organization' in result.msg"
- "result.total_results == 0"
- name: Delete an SSH credential
tower_credential:
@ -750,5 +752,7 @@
- assert:
that:
- result is failed
- "result.msg =='The organizations test-non-existing-org was not found on the Tower server'"
- "result is failed"
- "result is not changed"
- "'test-non-existing-org' in result.msg"
- "result.total_results == 0"

View File

@ -51,8 +51,10 @@
- assert:
that:
- "result.msg =='Failed to update the group, inventory not found: The requested object could not be found.' or
result.msg =='The inventories test-non-existing-inventory was not found on the Tower server'"
- "result is failed"
- "result is not changed"
- "'test-non-existing-inventory' in result.msg"
- "result.total_results == 0"
- name: add hosts
tower_host:

View File

@ -46,5 +46,6 @@
- assert:
that:
- "result.msg =='The inventories test-non-existing-inventory was not found on the Tower server' or
result.msg =='Failed to update host, inventory not found: The requested object could not be found.'"
- "result is failed"
- "'test-non-existing-inventory' in result.msg"
- "result.total_results == 0"

View File

@ -119,9 +119,11 @@
- assert:
that:
- "result is failed"
- "result is not changed"
- "result.msg =='Failed to update inventory, organization not found: The requested object could not be found.'
or result.msg =='The organizations test-non-existing-org was not found on the Tower server'"
- "'test-non-existing-org' in result.msg"
- "result.total_results == 0"
always:
- name: Delete Inventories
tower_inventory:

View File

@ -29,16 +29,16 @@
- name: Check module fails with correct msg
tower_job_launch:
job_template: "Non Existing Job Template"
inventory: "Test Inventory"
credential: "Test Credential"
job_template: "Non_Existing_Job_Template"
inventory: "Demo Inventory"
register: result
ignore_errors: true
- assert:
that:
- "result.msg =='Unable to launch job, job_template/Non Existing Job Template was not found: The requested object could not be found.'
or result.msg == 'The inventories Test Inventory was not found on the Tower server'"
- "result is failed"
- "result is not changed"
- "'Non_Existing_Job_Template' in result.msg"
- name: Create a Job Template for testing prompt on launch
tower_job_template:

View File

@ -12,13 +12,16 @@
- name: Check module fails with correct msg
tower_label:
name: "Test Label"
organization: "Non existing org"
organization: "Non_existing_org"
state: present
register: result
ignore_errors: true
- assert:
that:
- "'Non existing org was not found on the Tower server' in result.msg"
- "result is failed"
- "result is not changed"
- "'Non_existing_org' in result.msg"
- "result.total_results == 0"
# TODO: Deleting labels doesn't seem to work currently

View File

@ -93,7 +93,7 @@
- name: Check module fails with correct msg when given non-existing org as param
tower_project:
name: "{{ project_name2 }}"
organization: Non Existing Org
organization: Non_Existing_Org
scm_type: git
scm_url: https://github.com/ansible/test-playbooks
scm_credential: "{{ cred_name }}"
@ -102,8 +102,10 @@
- assert:
that:
- "result.msg == 'The organizations Non Existing Org was not found on the Tower server' or
result.msg == 'Failed to update project, organization not found: Non Existing Org'"
- "result is failed"
- "result is not changed"
- "'Non_Existing_Org' in result.msg"
- "result.total_results == 0"
- name: Check module fails with correct msg when given non-existing credential as param
tower_project:
@ -111,14 +113,16 @@
organization: "{{ org_name }}"
scm_type: git
scm_url: https://github.com/ansible/test-playbooks
scm_credential: Non Existing Credential
scm_credential: Non_Existing_Credential
register: result
ignore_errors: true
- assert:
that:
- "result.msg =='The credentials Non Existing Credential was not found on the Tower server' or
result.msg =='Failed to update project, credential not found: Non Existing Credential'"
- "result is failed"
- "result is not changed"
- "'Non_Existing_Credential' in result.msg"
- "result.total_results == 0"
- name: Create a git project without credentials without waiting
tower_project:

View File

@ -6,7 +6,7 @@
- name: Attempt to add a Tower team to a non-existant Organization
tower_team:
name: Test Team
organization: Missing Organization
organization: Missing_Organization
state: present
register: result
ignore_errors: true
@ -14,9 +14,10 @@
- name: Assert a meaningful error was provided for the failed Tower team creation
assert:
that:
- result is failed
- "result.msg =='Failed to update team, organization not found: The requested object could not be found.' or
result.msg =='The organizations Missing Organization was not found on the Tower server'"
- "result is failed"
- "result is not changed"
- "'Missing_Organization' in result.msg"
- "result.total_results == 0"
- name: Create a Tower team
tower_team:
@ -42,12 +43,15 @@
- name: Check module fails with correct msg
tower_team:
name: "{{ team_name }}"
organization: Non Existing Org
organization: Non_Existing_Org
state: present
register: result
ignore_errors: true
- assert:
- name: Lookup of the related organization should cause a failure
assert:
that:
- "result.msg =='Failed to update team, organization not found: The requested object could not be found.' or
result.msg =='The organizations Non Existing Org was not found on the Tower server'"
- "result is failed"
- "result is not changed"
- "'Non_Existing_Org' in result.msg"
- "result.total_results == 0"

View File

@ -207,13 +207,16 @@
- name: Check module fails with correct msg
tower_workflow_job_template:
name: "{{ wfjt_name }}"
organization: Non Existing Organization
organization: Non_Existing_Organization
register: result
ignore_errors: true
- assert:
that:
- "'The organizations Non Existing Organization was not found' in result.msg"
- "result is failed"
- "result is not changed"
- "'Non_Existing_Organization' in result.msg"
- "result.total_results == 0"
- name: Delete the Job Template
tower_job_template: