From 7dfa95761990cb5901803b04db2cb3240d5f6142 Mon Sep 17 00:00:00 2001 From: beeankha Date: Mon, 2 Nov 2020 15:08:55 -0500 Subject: [PATCH 1/6] Change import/export logging detection to detect errors vs warnings --- awx_collection/plugins/modules/tower_export.py | 16 +++++++++------- awx_collection/plugins/modules/tower_import.py | 16 +++++++++------- 2 files changed, 18 insertions(+), 14 deletions(-) diff --git a/awx_collection/plugins/modules/tower_export.py b/awx_collection/plugins/modules/tower_export.py index bd951d1744..ac7ff5a499 100644 --- a/awx_collection/plugins/modules/tower_export.py +++ b/awx_collection/plugins/modules/tower_export.py @@ -82,9 +82,11 @@ EXAMPLES = ''' - name: Export all tower assets tower_export: all: True + - name: Export all inventories tower_export: inventory: 'all' + - name: Export a job template named "My Template" and all Credentials tower_export: job_template: "My Template" @@ -135,27 +137,27 @@ def main(): # Otherwise we take either the string or None (if the parameter was not passed) to get one or no items export_args[resource] = module.params.get(resource) - # Currently the import process does not return anything on error - # It simply just logs to pythons logger - # Setup a log gobbler to get error messages from import_assets + # Currently the export process does not return anything on error + # It simply just logs to Python's logger + # Set up a log gobbler to get error messages from export_assets log_capture_string = StringIO() ch = logging.StreamHandler(log_capture_string) for logger_name in ['awxkit.api.pages.api', 'awxkit.api.pages.page']: logger = logging.getLogger(logger_name) - logger.setLevel(logging.WARNING) - ch.setLevel(logging.WARNING) + logger.setLevel(logging.ERROR) + ch.setLevel(logging.ERROR) logger.addHandler(ch) log_contents = '' - # Run the import process + # Run the export process try: module.json_output['assets'] = module.get_api_v2_object().export_assets(**export_args) module.exit_json(**module.json_output) except Exception as e: module.fail_json(msg="Failed to export assets {0}".format(e)) finally: - # Finally consume the logs incase there were any errors and die if there were + # Finally, consume the logs in case there were any errors and die if there were log_contents = log_capture_string.getvalue() log_capture_string.close() if log_contents != '': diff --git a/awx_collection/plugins/modules/tower_import.py b/awx_collection/plugins/modules/tower_import.py index a39a98a5e3..6b3282a847 100644 --- a/awx_collection/plugins/modules/tower_import.py +++ b/awx_collection/plugins/modules/tower_import.py @@ -38,7 +38,7 @@ EXAMPLES = ''' - name: Export all assets tower_export: all: True - registeR: export_output + register: export_output - name: Import all tower assets from our export tower_import: @@ -51,7 +51,7 @@ EXAMPLES = ''' from ..module_utils.tower_awxkit import TowerAWXKitModule -# These two lines are not needed if awxkit changes to do progamatic notifications on issues +# These two lines are not needed if awxkit changes to do programatic notifications on issues from ansible.module_utils.six.moves import StringIO import logging @@ -76,13 +76,15 @@ def main(): module.fail_json(msg="Your version of awxkit does not appear to have import/export") # Currently the import process does not return anything on error - # It simply just logs to pythons logger - # Setup a log gobbler to get error messages from import_assets + # It simply just logs to Python's logger + # Set up a log gobbler to get error messages from import_assets logger = logging.getLogger('awxkit.api.pages.api') - logger.setLevel(logging.WARNING) + logger.setLevel(logging.ERROR) + log_capture_string = StringIO() ch = logging.StreamHandler(log_capture_string) - ch.setLevel(logging.WARNING) + ch.setLevel(logging.ERROR) + logger.addHandler(ch) log_contents = '' @@ -92,7 +94,7 @@ def main(): except Exception as e: module.fail_json(msg="Failed to import assets {0}".format(e)) finally: - # Finally consume the logs incase there were any errors and die if there were + # Finally, consume the logs in case there were any errors and die if there were log_contents = log_capture_string.getvalue() log_capture_string.close() if log_contents != '': From 439302b38e983e925db04f09a263fb6d2a5314a1 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Tue, 3 Nov 2020 14:26:31 -0500 Subject: [PATCH 2/6] pin known working collections in 3.8.0 --- requirements/collections_requirements.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/requirements/collections_requirements.yml b/requirements/collections_requirements.yml index 2a8f46fa24..ca24924d82 100644 --- a/requirements/collections_requirements.yml +++ b/requirements/collections_requirements.yml @@ -1,12 +1,22 @@ --- collections: - name: awx.awx + version: 15.0.1 # almost all versions should work - name: azure.azcollection + version: 1.2.0 - name: amazon.aws + version: 1.2.1 - name: theforeman.foreman + version: 1.4.0 - name: google.cloud + version: 1.0.1 - name: openstack.cloud + version: 1.2.0 - name: community.vmware + version: 1.3.0 - name: ovirt.ovirt + version: 1.2.1 # see: https://github.com/ansible/tower/issues/4561 - name: community.kubernetes # required for isolated management playbooks + version: 1.1.1 - name: ansible.posix # required for isolated management playbooks + version: 1.1.1 From 28a70ced56eefd4e343260ed0ff3bc4a10488c3a Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Wed, 4 Nov 2020 15:18:12 -0500 Subject: [PATCH 3/6] Put everyone in the 'anonymous' pendo bucket regardless of setting We no longer have the key detailed accounts are indexed on. --- .../authenticationServices/pendo.service.js | 24 ++----------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/awx/ui/client/src/login/authenticationServices/pendo.service.js b/awx/ui/client/src/login/authenticationServices/pendo.service.js index a7296f6d8b..cab17baea3 100644 --- a/awx/ui/client/src/login/authenticationServices/pendo.service.js +++ b/awx/ui/client/src/login/authenticationServices/pendo.service.js @@ -29,30 +29,10 @@ export default ['$rootScope', 'Rest', 'GetBasePath', 'ProcessErrors', '$q', 'Con } }; - if (config.analytics_status === 'detailed') { - this.setDetailed(options, config); - } else if (config.analytics_status === 'anonymous') { - this.setAnonymous(options); - } - - return options; - }, - - // Detailed mode sends: - // VisitorId: userid+hash of license_key - // AccountId: hash of license_key from license - setDetailed: function(options, config) { - // config.deployment_id is a hash of the tower license_key - options.visitor.id = $rootScope.current_user.id + '@' + config.deployment_id; - options.account.id = config.deployment_id; - }, - - // Anonymous mode sends: - // VisitorId: - // AccountId: - setAnonymous: function (options) { options.visitor.id = 0; options.account.id = "tower.ansible.com"; + + return options; }, setRole: function(options) { From 662ee6fa367bd9fc921e4c2189d2775f8242974f Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 5 Nov 2020 12:29:06 -0500 Subject: [PATCH 4/6] update a janky old test UJT list views don't work this way anymore after a929e82060121371423ced061b10758c905d29cc --- awx/main/tests/functional/api/test_rbac_displays.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/awx/main/tests/functional/api/test_rbac_displays.py b/awx/main/tests/functional/api/test_rbac_displays.py index 4180647d44..d0a0cb4f98 100644 --- a/awx/main/tests/functional/api/test_rbac_displays.py +++ b/awx/main/tests/functional/api/test_rbac_displays.py @@ -282,10 +282,6 @@ def test_prefetch_ujt_project_capabilities(alice, project, job_template, mocker) list_serializer.child.to_representation(project) assert 'capability_map' not in list_serializer.child.context - # Models for which the prefetch is valid for do - list_serializer.child.to_representation(job_template) - assert set(list_serializer.child.context['capability_map'][job_template.id].keys()) == set(('copy', 'edit', 'start')) - @pytest.mark.django_db def test_prefetch_group_capabilities(group, rando): From 0b701b3b241d1387de87e037e30c777c39a9f572 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 5 Nov 2020 07:47:28 -0500 Subject: [PATCH 5/6] entirely remove CloudForms inventory sources instead of converting --- .../0117_v400_remove_cloudforms_inventory.py | 16 +++++-- awx/main/migrations/_inventory_source.py | 42 ------------------- awx/main/models/credential/__init__.py | 27 ------------ 3 files changed, 12 insertions(+), 73 deletions(-) diff --git a/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py b/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py index 8c33318755..fafd85e05b 100644 --- a/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py +++ b/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py @@ -1,11 +1,19 @@ # Generated by Django 2.2.11 on 2020-05-01 13:25 from django.db import migrations, models -from awx.main.migrations._inventory_source import create_scm_script_substitute +from awx.main.utils.common import set_current_apps -def convert_cloudforms_to_scm(apps, schema_editor): - create_scm_script_substitute(apps, 'cloudforms') +def delete_cloudforms_inv_source(apps, schema_editor): + set_current_apps(apps) + InventorySource = apps.get_model('main', 'InventorySource') + InventoryUpdate = apps.get_model('main', 'InventoryUpdate') + CredentialType = apps.get_model('main', 'CredentialType') + InventoryUpdate.objects.filter(inventory_source__source='cloudforms').delete() + InventorySource.objects.filter(source='cloudforms').delete() + ct = CredentialType.objects.filter(namespace='cloudforms').first() + ct.credentials.all().delete() + ct.delete() class Migration(migrations.Migration): @@ -15,7 +23,7 @@ class Migration(migrations.Migration): ] operations = [ - migrations.RunPython(convert_cloudforms_to_scm), + migrations.RunPython(delete_cloudforms_inv_source), migrations.AlterField( model_name='inventorysource', name='source', diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index ed79606587..d5107d66b2 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -89,45 +89,3 @@ def back_out_new_instance_id(apps, source, new_id): logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format( modified_ct, source )) - - -def create_scm_script_substitute(apps, source): - """Only applies for cloudforms in practice, but written generally. - Given a source type, this will replace all inventory sources of that type - with SCM inventory sources that source the script from Ansible core - """ - # the revision in the Ansible 2.9 stable branch this project will start out as - # it can still be updated manually later (but staying within 2.9 branch), if desired - ansible_rev = '6f83b9aff42331e15c55a171de0a8b001208c18c' - InventorySource = apps.get_model('main', 'InventorySource') - ContentType = apps.get_model('contenttypes', 'ContentType') - Project = apps.get_model('main', 'Project') - if not InventorySource.objects.filter(source=source).exists(): - logger.debug('No sources of type {} to migrate'.format(source)) - return - proj_name = 'Replacement project for {} type sources - {}'.format(source, uuid4()) - right_now = now() - project = Project.objects.create( - name=proj_name, - created=right_now, - modified=right_now, - description='Created by migration', - polymorphic_ctype=ContentType.objects.get(model='project'), - # project-specific fields - scm_type='git', - scm_url='https://github.com/ansible/ansible.git', - scm_branch='stable-2.9', - scm_revision=ansible_rev - ) - ct = 0 - for inv_src in InventorySource.objects.filter(source=source).iterator(): - inv_src.source = 'scm' - inv_src.source_project = project - inv_src.source_path = 'contrib/inventory/{}.py'.format(source) - inv_src.scm_last_revision = ansible_rev - inv_src.save(update_fields=['source', 'source_project', 'source_path', 'scm_last_revision']) - logger.debug('Changed inventory source {} to scm type'.format(inv_src.pk)) - ct += 1 - if ct: - logger.info('Changed total of {} inventory sources from {} type to scm'.format(ct, source)) - diff --git a/awx/main/models/credential/__init__.py b/awx/main/models/credential/__init__.py index df12177aae..66db962430 100644 --- a/awx/main/models/credential/__init__.py +++ b/awx/main/models/credential/__init__.py @@ -881,33 +881,6 @@ ManagedCredentialType( } ) -ManagedCredentialType( - namespace='cloudforms', - kind='cloud', - name=ugettext_noop('Red Hat CloudForms'), - managed_by_tower=True, - inputs={ - 'fields': [{ - 'id': 'host', - 'label': ugettext_noop('CloudForms URL'), - 'type': 'string', - 'help_text': ugettext_noop('Enter the URL for the virtual machine that ' - 'corresponds to your CloudForms instance. ' - 'For example, https://cloudforms.example.org') - }, { - 'id': 'username', - 'label': ugettext_noop('Username'), - 'type': 'string' - }, { - 'id': 'password', - 'label': ugettext_noop('Password'), - 'type': 'string', - 'secret': True, - }], - 'required': ['host', 'username', 'password'], - } -) - ManagedCredentialType( namespace='gce', kind='cloud', From 52d9fbce73f539236a544d286e453fb47527c3f1 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 5 Nov 2020 12:19:12 -0500 Subject: [PATCH 6/6] update cloudforms-specific tests --- .../0117_v400_remove_cloudforms_inventory.py | 14 +----- awx/main/migrations/_inventory_source.py | 14 ++++++ .../tests/functional/api/test_credential.py | 27 ---------- awx/main/tests/functional/test_credential.py | 1 - .../test_inventory_source_migration.py | 50 ++++++++++++++----- 5 files changed, 52 insertions(+), 54 deletions(-) diff --git a/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py b/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py index fafd85e05b..9a94c6b02b 100644 --- a/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py +++ b/awx/main/migrations/0117_v400_remove_cloudforms_inventory.py @@ -1,19 +1,7 @@ # Generated by Django 2.2.11 on 2020-05-01 13:25 from django.db import migrations, models -from awx.main.utils.common import set_current_apps - - -def delete_cloudforms_inv_source(apps, schema_editor): - set_current_apps(apps) - InventorySource = apps.get_model('main', 'InventorySource') - InventoryUpdate = apps.get_model('main', 'InventoryUpdate') - CredentialType = apps.get_model('main', 'CredentialType') - InventoryUpdate.objects.filter(inventory_source__source='cloudforms').delete() - InventorySource.objects.filter(source='cloudforms').delete() - ct = CredentialType.objects.filter(namespace='cloudforms').first() - ct.credentials.all().delete() - ct.delete() +from awx.main.migrations._inventory_source import delete_cloudforms_inv_source class Migration(migrations.Migration): diff --git a/awx/main/migrations/_inventory_source.py b/awx/main/migrations/_inventory_source.py index d5107d66b2..c53a18f035 100644 --- a/awx/main/migrations/_inventory_source.py +++ b/awx/main/migrations/_inventory_source.py @@ -5,6 +5,7 @@ from uuid import uuid4 from django.utils.encoding import smart_text from django.utils.timezone import now +from awx.main.utils.common import set_current_apps from awx.main.utils.common import parse_yaml_or_json logger = logging.getLogger('awx.main.migrations') @@ -89,3 +90,16 @@ def back_out_new_instance_id(apps, source, new_id): logger.info('Reverse migrated instance ID for {} hosts imported by {} source'.format( modified_ct, source )) + + +def delete_cloudforms_inv_source(apps, schema_editor): + set_current_apps(apps) + InventorySource = apps.get_model('main', 'InventorySource') + InventoryUpdate = apps.get_model('main', 'InventoryUpdate') + CredentialType = apps.get_model('main', 'CredentialType') + InventoryUpdate.objects.filter(inventory_source__source='cloudforms').delete() + InventorySource.objects.filter(source='cloudforms').delete() + ct = CredentialType.objects.filter(namespace='cloudforms').first() + if ct: + ct.credentials.all().delete() + ct.delete() diff --git a/awx/main/tests/functional/api/test_credential.py b/awx/main/tests/functional/api/test_credential.py index 9a534a8897..e8e7b4b271 100644 --- a/awx/main/tests/functional/api/test_credential.py +++ b/awx/main/tests/functional/api/test_credential.py @@ -675,33 +675,6 @@ def test_net_create_ok(post, organization, admin): assert cred.inputs['authorize'] is True -# -# Cloudforms Credentials -# -@pytest.mark.django_db -def test_cloudforms_create_ok(post, organization, admin): - params = { - 'credential_type': 1, - 'name': 'Best credential ever', - 'inputs': { - 'host': 'some_host', - 'username': 'some_username', - 'password': 'some_password', - } - } - cloudforms = CredentialType.defaults['cloudforms']() - cloudforms.save() - params['organization'] = organization.id - response = post(reverse('api:credential_list'), params, admin) - assert response.status_code == 201 - - assert Credential.objects.count() == 1 - cred = Credential.objects.all()[:1].get() - assert cred.inputs['host'] == 'some_host' - assert cred.inputs['username'] == 'some_username' - assert decrypt_field(cred, 'password') == 'some_password' - - # # GCE Credentials # diff --git a/awx/main/tests/functional/test_credential.py b/awx/main/tests/functional/test_credential.py index 684f9dd5a7..27f67b96f4 100644 --- a/awx/main/tests/functional/test_credential.py +++ b/awx/main/tests/functional/test_credential.py @@ -79,7 +79,6 @@ def test_default_cred_types(): 'aws', 'azure_kv', 'azure_rm', - 'cloudforms', 'conjur', 'galaxy_api_token', 'gce', diff --git a/awx/main/tests/functional/test_inventory_source_migration.py b/awx/main/tests/functional/test_inventory_source_migration.py index ecea2f0408..2b1e089392 100644 --- a/awx/main/tests/functional/test_inventory_source_migration.py +++ b/awx/main/tests/functional/test_inventory_source_migration.py @@ -5,7 +5,7 @@ from awx.main.migrations import _inventory_source as invsrc from django.apps import apps -from awx.main.models import InventorySource +from awx.main.models import InventorySource, InventoryUpdate, ManagedCredentialType, CredentialType, Credential @pytest.mark.parametrize('vars,id_var,result', [ @@ -42,16 +42,40 @@ def test_apply_new_instance_id(inventory_source): @pytest.mark.django_db -def test_replacement_scm_sources(inventory): - inv_source = InventorySource.objects.create( - name='test', - inventory=inventory, - organization=inventory.organization, - source='ec2' +def test_cloudforms_inventory_removal(inventory): + ManagedCredentialType( + name='Red Hat CloudForms', + namespace='cloudforms', + kind='cloud', + managed_by_tower=True, + inputs={}, ) - invsrc.create_scm_script_substitute(apps, 'ec2') - inv_source.refresh_from_db() - assert inv_source.source == 'scm' - assert inv_source.source_project - project = inv_source.source_project - assert 'Replacement project for' in project.name + CredentialType.defaults['cloudforms']().save() + cloudforms = CredentialType.objects.get(namespace='cloudforms') + Credential.objects.create( + name='test', + credential_type=cloudforms, + ) + + for source in ('ec2', 'cloudforms'): + i = InventorySource.objects.create( + name='test', + inventory=inventory, + organization=inventory.organization, + source=source, + ) + InventoryUpdate.objects.create( + name='test update', + inventory_source=i, + source=source, + ) + assert Credential.objects.count() == 1 + assert InventorySource.objects.count() == 2 # ec2 + cf + assert InventoryUpdate.objects.count() == 2 # ec2 + cf + invsrc.delete_cloudforms_inv_source(apps, None) + assert InventorySource.objects.count() == 1 # ec2 + assert InventoryUpdate.objects.count() == 1 # ec2 + assert InventorySource.objects.first().source == 'ec2' + assert InventoryUpdate.objects.first().source == 'ec2' + assert Credential.objects.count() == 0 + assert CredentialType.objects.filter(namespace='cloudforms').exists() is False