From 0e6c14e707e41accf8b0cbbdd0f9f41f008e78b6 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Thu, 18 Feb 2021 17:56:01 -0500 Subject: [PATCH 01/13] create performance logger to log api response time this middleware allready existed, and we were trying to log this data but it was not working. Hope is these logs will be able to be shipped via external logging and we could use kibana to track response time of different endpoints --- awx/main/middleware.py | 5 ++++- awx/settings/defaults.py | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/awx/main/middleware.py b/awx/main/middleware.py index 8bfd273811..2d509c9c61 100644 --- a/awx/main/middleware.py +++ b/awx/main/middleware.py @@ -45,7 +45,10 @@ class TimingMiddleware(threading.local, MiddlewareMixin): response['X-API-Total-Time'] = '%0.3fs' % total_time if settings.AWX_REQUEST_PROFILE: response['X-API-Profile-File'] = self.prof.stop() - perf_logger.info('api response times', extra=dict(python_objects=dict(request=request, response=response))) + perf_logger.info( + f'request: {request}, response_time: {response["X-API-Total-Time"]}', + extra=dict(python_objects=dict(request=request, response=response, X_API_TOTAL_TIME=response["X-API-Total-Time"])) + ) return response diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index af1590862c..f995ad6000 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1044,6 +1044,11 @@ LOGGING = { 'level': 'INFO', 'propagate': False }, + 'awx.analytics.performance': { + 'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], + 'level': 'DEBUG', + 'propagate': False + }, 'awx.analytics.job_lifecycle': { 'handlers': ['console', 'job_lifecycle'], 'level': 'DEBUG', From 88a91cfeba50e1a7a288ed8b616f0ca5d7da3f95 Mon Sep 17 00:00:00 2001 From: Elijah DeLee Date: Thu, 4 Mar 2021 11:09:32 -0500 Subject: [PATCH 02/13] pin websocket-client lib They've made breaking changes that is going to take some deeper investigation to update awxkit to use This is only used for development purposes, and should have not impact on the "awx" cli entry point --- awxkit/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awxkit/setup.py b/awxkit/setup.py index 6e357a800f..b446ba0a90 100644 --- a/awxkit/setup.py +++ b/awxkit/setup.py @@ -70,7 +70,7 @@ setup( python_requires=">=3.6", extras_require={ 'formatting': ['jq'], - 'websockets': ['websocket-client>0.54.0'], + 'websockets': ['websocket-client==0.57.0'], 'crypto': ['cryptography'] }, license='Apache 2.0', From 097f465f397e6ff1ac3a4e8edb1525b5e1172c12 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Mon, 8 Mar 2021 17:03:38 -0500 Subject: [PATCH 03/13] Fix the RBAC for attaching an EE to various objects - Organization.default_environment - Project.default_environment - JobTemplate.execution_environment - WorkflowJobTemplate.execution_environment System jobs are not editable by anyone other than a system admin, so we don't need to check. It appears that unified job templates can't be created or edited outside of the endpoints for the specific types. --- awx/main/access.py | 49 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/awx/main/access.py b/awx/main/access.py index d2a2aa9f3b..cdb0c0eb53 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -777,6 +777,11 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): @check_superuser def can_change(self, obj, data): + if data and data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return self.user in obj.admin_role def can_delete(self, obj): @@ -1385,14 +1390,29 @@ class ProjectAccess(NotificationAttachMixin, BaseAccess): def can_add(self, data): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'project_admin_role').exists() - return (self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and - self.check_related('credential', Credential, data, role_field='use_role')) + + if data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + + return ( + self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and + self.check_related('credential', Credential, data, role_field='use_role') + ) @check_superuser def can_change(self, obj, data): - return (self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and - self.user in obj.admin_role and - self.check_related('credential', Credential, data, obj=obj, role_field='use_role')) + if data and data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data, obj=obj) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + + return ( + self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and + self.user in obj.admin_role and + self.check_related('credential', Credential, data, obj=obj, role_field='use_role') + ) @check_superuser def can_start(self, obj, validate_license=True): @@ -1497,6 +1517,10 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.user not in inventory.use_role: return False + ee = get_value(ExecutionEnvironment, 'execution_environment') + if ee and not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + project = get_value(Project, 'project') # If the user has admin access to the project (as an org admin), should # be able to proceed without additional checks. @@ -1544,6 +1568,11 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.changes_are_non_sensitive(obj, data): return True + if data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + for required_field, cls in (('inventory', Inventory), ('project', Project)): is_mandatory = True if not getattr(obj, '{}_id'.format(required_field)): @@ -1974,6 +2003,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'workflow_admin_role').exists() + if data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return ( self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True) and self.check_related('inventory', Inventory, data, role_field='use_role') @@ -2023,6 +2057,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.user.is_superuser: return True + if data and data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return ( self.check_related('organization', Organization, data, role_field='workflow_admin_role', obj=obj) and self.check_related('inventory', Inventory, data, role_field='use_role', obj=obj) and From 80023017a23c2c5a33d2c3da01fb3a72c34e93b1 Mon Sep 17 00:00:00 2001 From: Nikhil Jain Date: Thu, 11 Mar 2021 19:14:50 +0530 Subject: [PATCH 04/13] fix the tower_user module to update the fields properly --- awx_collection/plugins/modules/tower_user.py | 14 ++++++------- awx_collection/test/awx/test_user.py | 21 ++++++++++++++++++++ 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/awx_collection/plugins/modules/tower_user.py b/awx_collection/plugins/modules/tower_user.py index 8f32c42666..b57266dbc0 100644 --- a/awx_collection/plugins/modules/tower_user.py +++ b/awx_collection/plugins/modules/tower_user.py @@ -149,19 +149,19 @@ def main(): # Create the data that gets sent for create and update new_fields = {} - if username: + if username is not None: new_fields['username'] = module.get_item_name(existing_item) if existing_item else username - if first_name: + if first_name is not None: new_fields['first_name'] = first_name - if last_name: + if last_name is not None: new_fields['last_name'] = last_name - if email: + if email is not None: new_fields['email'] = email - if is_superuser: + if is_superuser is not None: new_fields['is_superuser'] = is_superuser - if is_system_auditor: + if is_system_auditor is not None: new_fields['is_system_auditor'] = is_system_auditor - if password: + if password is not None: new_fields['password'] = password # If the state was present and we can let the module build or update the existing item, this will return on its own diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index 6c49f04121..1877d39df7 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -57,3 +57,24 @@ def test_update_password_on_create(run_module, admin_user, mock_auth_stuff): assert not result.get('failed', False), result.get('msg', result) assert not result.get('changed') + + +@pytest.mark.django_db +def test_update_user(run_module, admin_user, mock_auth_stuff): +    result = run_module('tower_user', dict( +        username='Bob', +        password='pass4word', +        is_system_auditor=True +    ), admin_user) +    assert not result.get('failed', False), result.get('msg', result) +    assert result.get('changed'), result + +    update_result =     result = run_module('tower_user', dict( +        username='Bob', +        password='pass4word', +        is_system_auditor=False +    ), admin_user) + +    assert update_result.get('changed') +    user = User.objects.get(id=result['id']) +    assert not user.is_system_auditor From 8e53453737fafa459849d0ee74105252078146c1 Mon Sep 17 00:00:00 2001 From: Nikhil Jain Date: Thu, 11 Mar 2021 19:35:53 +0530 Subject: [PATCH 05/13] remove the extra result in test --- awx_collection/test/awx/test_user.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index 1877d39df7..34adfaa6e9 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -69,7 +69,7 @@ def test_update_user(run_module, admin_user, mock_auth_stuff):     assert not result.get('failed', False), result.get('msg', result)     assert result.get('changed'), result -    update_result =     result = run_module('tower_user', dict( +    update_result = run_module('tower_user', dict(         username='Bob',         password='pass4word',         is_system_auditor=False From 53da8e0775c0aeea1cb2b2b64c426ba907064e7b Mon Sep 17 00:00:00 2001 From: Nikhil Jain Date: Thu, 11 Mar 2021 19:50:11 +0530 Subject: [PATCH 06/13] removing some invalid chars --- awx_collection/test/awx/test_user.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index 34adfaa6e9..a530e63d90 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -61,7 +61,7 @@ def test_update_password_on_create(run_module, admin_user, mock_auth_stuff): @pytest.mark.django_db def test_update_user(run_module, admin_user, mock_auth_stuff): -    result = run_module('tower_user', dict( + result = run_module('tower_user', dict(         username='Bob',         password='pass4word',         is_system_auditor=True @@ -69,9 +69,8 @@ def test_update_user(run_module, admin_user, mock_auth_stuff):     assert not result.get('failed', False), result.get('msg', result)     assert result.get('changed'), result -    update_result = run_module('tower_user', dict( + update_result = run_module('tower_user', dict(         username='Bob', -        password='pass4word',         is_system_auditor=False     ), admin_user) From 2aa30226f43fd64a3278b0d33e870293bac6520f Mon Sep 17 00:00:00 2001 From: Nikhil Jain Date: Thu, 11 Mar 2021 20:07:31 +0530 Subject: [PATCH 07/13] removing some invalid chars --- awx_collection/test/awx/test_user.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index a530e63d90..6a0dfa123d 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -62,18 +62,18 @@ def test_update_password_on_create(run_module, admin_user, mock_auth_stuff): @pytest.mark.django_db def test_update_user(run_module, admin_user, mock_auth_stuff): result = run_module('tower_user', dict( -        username='Bob', -        password='pass4word', -        is_system_auditor=True -    ), admin_user) -    assert not result.get('failed', False), result.get('msg', result) -    assert result.get('changed'), result + username='Bob', + password='pass4word', + is_system_auditor=True + ), admin_user) + assert not result.get('failed', False), result.get('msg', result) + assert result.get('changed'), result update_result = run_module('tower_user', dict( -        username='Bob', -        is_system_auditor=False -    ), admin_user) + username='Bob', + is_system_auditor=False + ), admin_user) -    assert update_result.get('changed') -    user = User.objects.get(id=result['id']) -    assert not user.is_system_auditor + assert update_result.get('changed') + user = User.objects.get(id=result['id']) + assert not user.is_system_auditor From 572c0fbb74b45da7ce1722ecb905886defa307f0 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 11 Mar 2021 11:13:22 -0500 Subject: [PATCH 08/13] clean stale dispatcher connections closer to post-fork see: https://github.com/ansible/awx/issues/9559 --- awx/conf/settings.py | 1 - awx/main/dispatch/worker/task.py | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 4b18e3d9f6..500b4408f5 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -352,7 +352,6 @@ class SettingsWrapper(UserSettingsHolder): self.cache.set_many(settings_to_cache, timeout=SETTING_CACHE_TIMEOUT) def _get_local(self, name, validate=True): - self.__clean_on_fork__() self._preload_cache() cache_key = Setting.get_cache_key(name) try: diff --git a/awx/main/dispatch/worker/task.py b/awx/main/dispatch/worker/task.py index 80a414770b..d71d62686f 100644 --- a/awx/main/dispatch/worker/task.py +++ b/awx/main/dispatch/worker/task.py @@ -6,6 +6,7 @@ import traceback from kubernetes.config import kube_config +from django.conf import settings from django_guid.middleware import GuidMiddleware from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown @@ -85,6 +86,7 @@ class TaskWorker(BaseWorker): 'task': u'awx.main.tasks.RunProjectUpdate' } ''' + settings.__clean_on_fork__() result = None try: result = self.run_callable(body) From e31fc372153f75abcd1110517c8670f3e4cba16b Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 11 Mar 2021 11:25:59 -0500 Subject: [PATCH 09/13] Make sure that EE foreign keys are polymorphic.SET_NULL Deleting EEs that had been attached to something was failing. --- .../0130_ee_polymorphic_set_null.py | 34 +++++++++++++++++++ awx/main/models/mixins.py | 4 +-- awx/main/models/organization.py | 3 +- awx/main/models/projects.py | 4 +-- 4 files changed, 40 insertions(+), 5 deletions(-) create mode 100644 awx/main/migrations/0130_ee_polymorphic_set_null.py diff --git a/awx/main/migrations/0130_ee_polymorphic_set_null.py b/awx/main/migrations/0130_ee_polymorphic_set_null.py new file mode 100644 index 0000000000..a9a0b63715 --- /dev/null +++ b/awx/main/migrations/0130_ee_polymorphic_set_null.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.16 on 2021-03-11 16:25 + +import awx.main.utils.polymorphic +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0129_unifiedjob_installed_collections'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='project', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run using this project.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='unifiedjob', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobs', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='unifiedjobtemplate', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobtemplates', to='main.ExecutionEnvironment'), + ), + ] diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 549c93607d..9df38b3fa4 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -22,7 +22,7 @@ from awx.main.models.base import prevent_search from awx.main.models.rbac import ( Role, RoleAncestorEntry, get_roles_on_resource ) -from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser +from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.utils.polymorphic import build_polymorphic_ctypes_map from awx.main.fields import JSONField, AskForField @@ -450,7 +450,7 @@ class ExecutionEnvironmentMixin(models.Model): null=True, blank=True, default=None, - on_delete=models.SET_NULL, + on_delete=polymorphic.SET_NULL, related_name='%(class)ss', help_text=_('The container image to be used for execution.'), ) diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index f0ecfea5c7..f087b8eb6b 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -27,6 +27,7 @@ from awx.main.models.rbac import ( ) from awx.main.models.unified_jobs import UnifiedJob from awx.main.models.mixins import ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin +from awx.main.utils import polymorphic __all__ = ['Organization', 'Team', 'Profile', 'UserSessionMembership'] @@ -66,7 +67,7 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi null=True, blank=True, default=None, - on_delete=models.SET_NULL, + on_delete=polymorphic.SET_NULL, related_name='+', help_text=_('The default execution environment for jobs run by this organization.'), ) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index c9bf87f408..5a613b6c21 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -35,7 +35,7 @@ from awx.main.models.mixins import ( CustomVirtualEnvMixin, RelatedJobsMixin ) -from awx.main.utils import update_scm_url +from awx.main.utils import update_scm_url, polymorphic from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook from awx.main.fields import ImplicitRoleField from awx.main.models.rbac import ( @@ -272,7 +272,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn null=True, blank=True, default=None, - on_delete=models.SET_NULL, + on_delete=polymorphic.SET_NULL, related_name='+', help_text=_('The default execution environment for jobs run using this project.'), ) From 588cb1e40339a2d3e9502f9db862db53525112a4 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Thu, 11 Mar 2021 08:43:49 -0500 Subject: [PATCH 10/13] fix some requirements updater breakage - remove requirements_ansible logic from the update script - removed the need for py2-specific system dependencies - update to the latest pip-tools and move to the new long format (https://github.com/jazzband/pip-tools/pull/1237) - fixed a few busted references to receptorctl @ devel --- docs/licenses/click.txt | 28 + requirements/README.md | 19 +- requirements/requirements.txt | 561 +++++++++++++----- requirements/requirements_git.txt | 2 +- requirements/updater.sh | 63 +- .../roles/dockerfile/templates/Dockerfile.j2 | 1 + 6 files changed, 467 insertions(+), 207 deletions(-) create mode 100644 docs/licenses/click.txt diff --git a/docs/licenses/click.txt b/docs/licenses/click.txt new file mode 100644 index 0000000000..d12a849186 --- /dev/null +++ b/docs/licenses/click.txt @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/requirements/README.md b/requirements/README.md index 22dfe8d62d..ab694dfbe3 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -1,15 +1,11 @@ # Dependency Management -The `requirements.txt` and `requirements_ansible.txt` files are generated from `requirements.in` and `requirements_ansible.in`, respectively, using `pip-tools` `pip-compile`. +The `requirements.txt` file is generated from `requirements.in`, using `pip-tools` `pip-compile`. ## How To Use Commands should be run from inside the `./requirements` directory of the awx repository. -Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed. The development container image should have all these. - -Even in the dev container, you may still have to dnf install `libpq-devel libcurl-devel`. - ### Upgrading or Adding Select Libraries If you need to add or upgrade one targeted library, then modify `requirements.in`, @@ -33,14 +29,6 @@ You can upgrade (`pip-compile --upgrade`) the dependencies by running `./updater.sh upgrade`. -## What The Script Does - -This script will: - - - Update `requirements.txt` based on `requirements.in` - - Update/generate `requirements_ansible.txt` based on `requirements_ansible.in` - - including an automated patch that adds `python_version < "3"` for Python 2 backward compatibility - ## Licenses and Source Files If any library has a change to its license with the upgrade, then the license for that library @@ -129,11 +117,6 @@ This breaks a very large amount of AWX code that assumes these fields are returned as dicts. Upgrading this library will require a refactor to accomidate this change. -### wheel - -azure-cli-core requires a version of wheel that is incompatible with -certain packages building with later versions of pip, so we override it. - ### pip and setuptools The offline installer needs to have functionality confirmed before upgrading these. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6256afd061..ba3cfe3087 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,135 +1,430 @@ -adal==1.2.2 # via msrestazure -aiohttp==3.6.2 # via -r /awx_devel/requirements/requirements.in -aioredis==1.3.1 # via channels-redis -# ansible-runner==1.4.7 # via -r /awx_devel/requirements/requirements.in -ansiconv==1.0.0 # via -r /awx_devel/requirements/requirements.in -asciichartpy==1.5.25 # via -r /awx_devel/requirements/requirements.in -asgiref==3.2.5 # via channels, channels-redis, daphne -async-timeout==3.0.1 # via aiohttp, aioredis -attrs==19.3.0 # via aiohttp, automat, jsonschema, service-identity, twisted -autobahn==20.12.3 # via -r /awx_devel/requirements/requirements.in, daphne -automat==20.2.0 # via twisted -azure-common==1.1.25 # via azure-keyvault -azure-keyvault==1.1.0 # via -r /awx_devel/requirements/requirements.in -azure-nspkg==3.0.2 # via azure-keyvault -cachetools==4.0.0 # via google-auth -# don't add certifi to our requirements, because we have our own specific implementation: https://github.com/ansible/system-certifi -cffi==1.14.0 # via cryptography -channels-redis==3.1.0 # via -r /awx_devel/requirements/requirements.in -channels==2.4.0 # via -r /awx_devel/requirements/requirements.in, channels-redis -chardet==3.0.4 # via aiohttp, requests -constantly==15.1.0 # via twisted -cryptography==2.9.2 # via -r /awx_devel/requirements/requirements.in, adal, autobahn, azure-keyvault, pyopenssl, service-identity, social-auth-core -daphne==2.4.1 # via -r /awx_devel/requirements/requirements.in, channels -defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core -dictdiffer==0.8.1 # via openshift -django-auth-ldap==2.1.0 # via -r /awx_devel/requirements/requirements.in -django-cors-headers==3.2.1 # via -r /awx_devel/requirements/requirements.in -django-crum==0.7.5 # via -r /awx_devel/requirements/requirements.in -django-extensions==2.2.9 # via -r /awx_devel/requirements/requirements.in -django-guid==2.2.0 # via -r /awx_devel/requirements/requirements.in -django-jsonfield==1.2.0 # via -r /awx_devel/requirements/requirements.in -django-oauth-toolkit==1.1.3 # via -r /awx_devel/requirements/requirements.in -django-pglocks==1.0.4 # via -r /awx_devel/requirements/requirements.in -django-polymorphic==2.1.2 # via -r /awx_devel/requirements/requirements.in -django-qsstats-magic==1.1.0 # via -r /awx_devel/requirements/requirements.in -django-radius==1.3.3 # via -r /awx_devel/requirements/requirements.in -django-redis==4.5.0 # via -r /awx_devel/requirements/requirements.in -django-solo==1.1.3 # via -r /awx_devel/requirements/requirements.in -django-split-settings==1.0.0 # via -r /awx_devel/requirements/requirements.in -django-taggit==1.2.0 # via -r /awx_devel/requirements/requirements.in -django==2.2.16 # via -r /awx_devel/requirements/requirements.in, channels, django-auth-ldap, django-cors-headers, django-crum, django-guid, django-jsonfield, django-oauth-toolkit, django-polymorphic, django-taggit, djangorestframework -djangorestframework-yaml==1.0.3 # via -r /awx_devel/requirements/requirements.in -djangorestframework==3.12.1 # via -r /awx_devel/requirements/requirements.in -docutils==0.16 # via python-daemon -future==0.16.0 # via django-radius -gitdb==4.0.2 # via gitpython -gitpython==3.1.7 # via -r /awx_devel/requirements/requirements.in -google-auth==1.11.3 # via kubernetes -hiredis==1.0.1 # via aioredis -hyperlink==20.0.1 # via autobahn, twisted -idna-ssl==1.1.0 # via aiohttp -idna==2.9 # via hyperlink, idna-ssl, requests, twisted, yarl -importlib-metadata==1.5.0 # via importlib-resources, irc, jsonschema -importlib-resources==1.4.0 # via jaraco.text -incremental==17.5.0 # via twisted -irc==18.0.0 # via -r /awx_devel/requirements/requirements.in -isodate==0.6.0 # via msrest, python3-saml -jaraco.classes==3.1.0 # via jaraco.collections -jaraco.collections==3.0.0 # via irc -jaraco.functools==3.0.0 # via irc, jaraco.text, tempora -jaraco.logging==3.0.0 # via irc -jaraco.stream==3.0.0 # via irc -jaraco.text==3.2.0 # via irc, jaraco.collections -jinja2==2.11.2 # via -r /awx_devel/requirements/requirements.in, openshift -json-log-formatter==0.3.0 # via -r /awx_devel/requirements/requirements.in -jsonschema==3.2.0 # via -r /awx_devel/requirements/requirements.in -kubernetes==11.0.0 # via openshift -lockfile==0.12.2 # via python-daemon -lxml==4.5.0 # via xmlsec -markdown==3.2.1 # via -r /awx_devel/requirements/requirements.in -markupsafe==1.1.1 # via jinja2 -more-itertools==8.2.0 # via irc, jaraco.classes, jaraco.functools -msgpack==1.0.0 # via channels-redis -msrest==0.6.11 # via azure-keyvault, msrestazure -msrestazure==0.6.3 # via azure-keyvault -multidict==4.7.5 # via aiohttp, yarl -netaddr==0.7.19 # via pyrad -oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core -openshift==0.11.0 # via -r /awx_devel/requirements/requirements.in -pexpect==4.7.0 # via -r /awx_devel/requirements/requirements.in, ansible-runner -pkgconfig==1.5.1 # via xmlsec -prometheus-client==0.7.1 # via -r /awx_devel/requirements/requirements.in -psutil==5.7.0 # via ansible-runner -psycopg2==2.8.4 # via -r /awx_devel/requirements/requirements.in -ptyprocess==0.6.0 # via pexpect -pyasn1-modules==0.2.8 # via google-auth, python-ldap, service-identity -pyasn1==0.4.8 # via pyasn1-modules, python-ldap, rsa, service-identity -pycparser==2.20 # via cffi -pygerduty==0.38.2 # via -r /awx_devel/requirements/requirements.in -pyhamcrest==2.0.2 # via twisted -pyjwt==1.7.1 # via adal, social-auth-core, twilio -pyopenssl==19.1.0 # via twisted -pyparsing==2.4.6 # via -r /awx_devel/requirements/requirements.in -pyrad==2.3 # via django-radius -pyrsistent==0.15.7 # via jsonschema -python-daemon==2.2.4 # via ansible-runner -python-dateutil==2.8.1 # via adal, kubernetes -python-ldap==3.3.1 # via -r /awx_devel/requirements/requirements.in, django-auth-ldap -python-string-utils==1.0.0 # via openshift -python3-openid==3.1.0 # via social-auth-core -python3-saml==1.9.0 # via -r /awx_devel/requirements/requirements.in -pytz==2019.3 # via django, irc, tempora, twilio -pyyaml==5.4.1 # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes -redis==3.4.1 # via -r /awx_devel/requirements/requirements.in, django-redis -requests-oauthlib==1.3.0 # via kubernetes, msrest, social-auth-core -requests==2.23.0 # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-oauthlib, slackclient, social-auth-core, twilio -rsa==4.0 # via google-auth -ruamel.yaml.clib==0.2.0 # via ruamel.yaml -ruamel.yaml==0.16.10 # via openshift -schedule==0.6.0 # via -r /awx_devel/requirements/requirements.in -service-identity==18.1.0 # via twisted -six==1.14.0 # via ansible-runner, automat, cryptography, django-extensions, django-pglocks, google-auth, isodate, jaraco.collections, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyopenssl, pyrad, pyrsistent, python-dateutil, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, twilio, websocket-client -slackclient==1.1.2 # via -r /awx_devel/requirements/requirements.in -smmap==3.0.1 # via gitdb -social-auth-app-django==3.1.0 # via -r /awx_devel/requirements/requirements.in -social-auth-core==3.3.1 # via -r /awx_devel/requirements/requirements.in, social-auth-app-django -sqlparse==0.3.1 # via django -tacacs_plus==1.0 # via -r /awx_devel/requirements/requirements.in -tempora==2.1.0 # via irc, jaraco.logging -twilio==6.37.0 # via -r /awx_devel/requirements/requirements.in -twisted[tls]==20.3.0 # via -r /awx_devel/requirements/requirements.in, daphne -txaio==20.12.1 # via autobahn -typing-extensions==3.7.4.1 # via aiohttp -urllib3==1.25.8 # via kubernetes, requests -uwsgi==2.0.18 # via -r /awx_devel/requirements/requirements.in -uwsgitop==0.11 # via -r /awx_devel/requirements/requirements.in -websocket-client==0.57.0 # via kubernetes, slackclient -xmlsec==1.3.3 # via python3-saml -yarl==1.4.2 # via aiohttp -zipp==3.1.0 # via importlib-metadata, importlib-resources -zope.interface==5.0.0 # via twisted +adal==1.2.2 + # via msrestazure +aiohttp==3.6.2 + # via -r /awx_devel/requirements/requirements.in +aioredis==1.3.1 + # via channels-redis +#ansible-runner==1.4.7 + # via + # -r /awx_devel/requirements/requirements.in + # -r /awx_devel/requirements/requirements_git.txt +ansiconv==1.0.0 + # via -r /awx_devel/requirements/requirements.in +asciichartpy==1.5.25 + # via -r /awx_devel/requirements/requirements.in +asgiref==3.2.5 + # via + # channels + # channels-redis + # daphne +async-timeout==3.0.1 + # via + # aiohttp + # aioredis +attrs==19.3.0 + # via + # aiohttp + # automat + # jsonschema + # service-identity + # twisted +autobahn==20.12.3 + # via + # -r /awx_devel/requirements/requirements.in + # daphne +automat==20.2.0 + # via twisted +azure-common==1.1.25 + # via azure-keyvault +azure-keyvault==1.1.0 + # via -r /awx_devel/requirements/requirements.in +azure-nspkg==3.0.2 + # via azure-keyvault +cachetools==4.0.0 + # via google-auth + # via + # -r /awx_devel/requirements/requirements_git.txt + # kubernetes + # msrest + # requests +cffi==1.14.0 + # via cryptography +channels-redis==3.1.0 + # via -r /awx_devel/requirements/requirements.in +channels==2.4.0 + # via + # -r /awx_devel/requirements/requirements.in + # channels-redis +chardet==3.0.4 + # via + # aiohttp + # requests +click==7.1.2 + # via receptorctl +constantly==15.1.0 + # via twisted +cryptography==2.9.2 + # via + # -r /awx_devel/requirements/requirements.in + # adal + # autobahn + # azure-keyvault + # pyopenssl + # service-identity + # social-auth-core +daphne==2.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # channels +defusedxml==0.6.0 + # via + # python3-openid + # python3-saml + # social-auth-core +dictdiffer==0.8.1 + # via openshift +django-auth-ldap==2.1.0 + # via -r /awx_devel/requirements/requirements.in +django-cors-headers==3.2.1 + # via -r /awx_devel/requirements/requirements.in +django-crum==0.7.5 + # via -r /awx_devel/requirements/requirements.in +django-extensions==2.2.9 + # via -r /awx_devel/requirements/requirements.in +django-guid==2.2.0 + # via -r /awx_devel/requirements/requirements.in +django-jsonfield==1.2.0 + # via -r /awx_devel/requirements/requirements.in +django-oauth-toolkit==1.1.3 + # via -r /awx_devel/requirements/requirements.in +django-pglocks==1.0.4 + # via -r /awx_devel/requirements/requirements.in +django-polymorphic==2.1.2 + # via -r /awx_devel/requirements/requirements.in +django-qsstats-magic==1.1.0 + # via -r /awx_devel/requirements/requirements.in +django-radius==1.3.3 + # via -r /awx_devel/requirements/requirements.in +django-redis==4.5.0 + # via -r /awx_devel/requirements/requirements.in +django-solo==1.1.3 + # via -r /awx_devel/requirements/requirements.in +django-split-settings==1.0.0 + # via -r /awx_devel/requirements/requirements.in +django-taggit==1.2.0 + # via -r /awx_devel/requirements/requirements.in +django==2.2.16 + # via + # -r /awx_devel/requirements/requirements.in + # channels + # django-auth-ldap + # django-cors-headers + # django-crum + # django-guid + # django-jsonfield + # django-oauth-toolkit + # django-polymorphic + # django-taggit + # djangorestframework +djangorestframework-yaml==1.0.3 + # via -r /awx_devel/requirements/requirements.in +djangorestframework==3.12.1 + # via -r /awx_devel/requirements/requirements.in +docutils==0.16 + # via python-daemon +future==0.16.0 + # via django-radius +gitdb==4.0.2 + # via gitpython +gitpython==3.1.7 + # via -r /awx_devel/requirements/requirements.in +google-auth==1.11.3 + # via kubernetes +hiredis==1.0.1 + # via aioredis +hyperlink==20.0.1 + # via + # autobahn + # twisted +idna-ssl==1.1.0 + # via aiohttp +idna==2.9 + # via + # hyperlink + # idna-ssl + # requests + # twisted + # yarl +importlib-metadata==1.5.0 + # via + # importlib-resources + # irc + # jsonschema +importlib-resources==1.4.0 + # via jaraco.text +incremental==17.5.0 + # via twisted +irc==18.0.0 + # via -r /awx_devel/requirements/requirements.in +isodate==0.6.0 + # via + # msrest + # python3-saml +jaraco.classes==3.1.0 + # via jaraco.collections +jaraco.collections==3.0.0 + # via irc +jaraco.functools==3.0.0 + # via + # irc + # jaraco.text + # tempora +jaraco.logging==3.0.0 + # via irc +jaraco.stream==3.0.0 + # via irc +jaraco.text==3.2.0 + # via + # irc + # jaraco.collections +jinja2==2.11.2 + # via + # -r /awx_devel/requirements/requirements.in + # openshift +json-log-formatter==0.3.0 + # via -r /awx_devel/requirements/requirements.in +jsonschema==3.2.0 + # via -r /awx_devel/requirements/requirements.in +kubernetes==11.0.0 + # via openshift +lockfile==0.12.2 + # via python-daemon +lxml==4.5.0 + # via xmlsec +markdown==3.2.1 + # via -r /awx_devel/requirements/requirements.in +markupsafe==1.1.1 + # via jinja2 +more-itertools==8.2.0 + # via + # irc + # jaraco.classes + # jaraco.functools +msgpack==1.0.0 + # via channels-redis +msrest==0.6.11 + # via + # azure-keyvault + # msrestazure +msrestazure==0.6.3 + # via azure-keyvault +multidict==4.7.5 + # via + # aiohttp + # yarl +netaddr==0.7.19 + # via pyrad +oauthlib==3.1.0 + # via + # django-oauth-toolkit + # requests-oauthlib + # social-auth-core +openshift==0.11.0 + # via -r /awx_devel/requirements/requirements.in +pexpect==4.7.0 + # via + # -r /awx_devel/requirements/requirements.in + # ansible-runner +pkgconfig==1.5.1 + # via xmlsec +prometheus-client==0.7.1 + # via -r /awx_devel/requirements/requirements.in +psutil==5.7.0 + # via ansible-runner +psycopg2==2.8.4 + # via -r /awx_devel/requirements/requirements.in +ptyprocess==0.6.0 + # via pexpect +pyasn1-modules==0.2.8 + # via + # google-auth + # python-ldap + # service-identity +pyasn1==0.4.8 + # via + # pyasn1-modules + # python-ldap + # rsa + # service-identity +pycparser==2.20 + # via cffi +pygerduty==0.38.2 + # via -r /awx_devel/requirements/requirements.in +pyhamcrest==2.0.2 + # via twisted +pyjwt==1.7.1 + # via + # adal + # social-auth-core + # twilio +pyopenssl==19.1.0 + # via twisted +pyparsing==2.4.6 + # via -r /awx_devel/requirements/requirements.in +pyrad==2.3 + # via django-radius +pyrsistent==0.15.7 + # via jsonschema +python-daemon==2.2.4 + # via ansible-runner +python-dateutil==2.8.1 + # via + # adal + # kubernetes + # receptorctl +python-ldap==3.3.1 + # via + # -r /awx_devel/requirements/requirements.in + # django-auth-ldap +python-string-utils==1.0.0 + # via openshift +python3-openid==3.1.0 + # via social-auth-core +python3-saml==1.9.0 + # via -r /awx_devel/requirements/requirements.in +pytz==2019.3 + # via + # django + # irc + # tempora + # twilio +pyyaml==5.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # ansible-runner + # djangorestframework-yaml + # kubernetes + # receptorctl + # via -r /awx_devel/requirements/requirements_git.txt +redis==3.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # django-redis +requests-oauthlib==1.3.0 + # via + # kubernetes + # msrest + # social-auth-core +requests==2.23.0 + # via + # -r /awx_devel/requirements/requirements.in + # adal + # azure-keyvault + # django-oauth-toolkit + # kubernetes + # msrest + # requests-oauthlib + # slackclient + # social-auth-core + # twilio +rsa==4.0 + # via google-auth +ruamel.yaml.clib==0.2.0 + # via ruamel.yaml +ruamel.yaml==0.16.10 + # via openshift +schedule==0.6.0 + # via -r /awx_devel/requirements/requirements.in +service-identity==18.1.0 + # via twisted +six==1.14.0 + # via + # ansible-runner + # automat + # cryptography + # django-extensions + # django-pglocks + # google-auth + # isodate + # jaraco.collections + # jaraco.logging + # jaraco.text + # jsonschema + # kubernetes + # openshift + # pygerduty + # pyopenssl + # pyrad + # pyrsistent + # python-dateutil + # slackclient + # social-auth-app-django + # social-auth-core + # tacacs-plus + # twilio + # websocket-client +slackclient==1.1.2 + # via -r /awx_devel/requirements/requirements.in +smmap==3.0.1 + # via gitdb +social-auth-app-django==3.1.0 + # via -r /awx_devel/requirements/requirements.in +social-auth-core==3.3.1 + # via + # -r /awx_devel/requirements/requirements.in + # social-auth-app-django +sqlparse==0.3.1 + # via django +tacacs_plus==1.0 + # via -r /awx_devel/requirements/requirements.in +tempora==2.1.0 + # via + # irc + # jaraco.logging +twilio==6.37.0 + # via -r /awx_devel/requirements/requirements.in +twisted[tls]==20.3.0 + # via + # -r /awx_devel/requirements/requirements.in + # daphne +txaio==20.12.1 + # via autobahn +typing-extensions==3.7.4.1 + # via aiohttp +urllib3==1.25.8 + # via + # kubernetes + # requests +uwsgi==2.0.18 + # via -r /awx_devel/requirements/requirements.in +uwsgitop==0.11 + # via -r /awx_devel/requirements/requirements.in +websocket-client==0.57.0 + # via + # kubernetes + # slackclient +xmlsec==1.3.3 + # via python3-saml +yarl==1.4.2 + # via aiohttp +zipp==3.1.0 + # via + # importlib-metadata + # importlib-resources +zope.interface==5.0.0 + # via twisted # The following packages are considered to be unsafe in a requirements file: -pip==19.3.1 # via -r /awx_devel/requirements/requirements.in -setuptools==41.6.0 # via -r /awx_devel/requirements/requirements.in, asciichartpy, google-auth, jsonschema, kubernetes, markdown, python-daemon, zope.interface +pip==19.3.1 + # via -r /awx_devel/requirements/requirements.in +setuptools==41.6.0 + # via + # -r /awx_devel/requirements/requirements.in + # asciichartpy + # google-auth + # jsonschema + # kubernetes + # markdown + # python-daemon + # receptorctl + # zope.interface diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 74f1fb4a9e..f2f3abaa7a 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,3 +1,3 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner -git+https://github.com/project-receptor/receptor.git@#egg=receptorctl&subdirectory=receptorctl +git+https://github.com/project-receptor/receptor.git@0.9.6#egg=receptorctl&subdirectory=receptorctl diff --git a/requirements/updater.sh b/requirements/updater.sh index c58f1a0f62..d184cf8796 100755 --- a/requirements/updater.sh +++ b/requirements/updater.sh @@ -2,71 +2,32 @@ set -ue requirements_in="$(readlink -f ./requirements.in)" -requirements_ansible_in="$(readlink -f ./requirements_ansible.in)" requirements="$(readlink -f ./requirements.txt)" requirements_git="$(readlink -f ./requirements_git.txt)" -requirements_ansible="$(readlink -f ./requirements_ansible.txt)" pip_compile="pip-compile --no-header --quiet -r --allow-unsafe" -check_prerequisites() { - for thing in patch awk python3 python2 virtualenv ; do - command -v $thing >/dev/null 2>&1 || { echo "$thing not installed or available. Please fix this before running." ; exit 1 ; } - done -} - _cleanup() { cd / test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}" } -install_deps() { - pip install pip --upgrade - pip install "pip-tools==5.4.0" # see https://github.com/jazzband/pip-tools/pull/1237 -} - -generate_requirements_v3() { - venv="./venv3" - python3 -m venv "${venv}" +generate_requirements() { + venv="`pwd`/venv" + echo $venv + /usr/bin/python3 -m venv "${venv}" # shellcheck disable=SC1090 - . "${venv}/bin/activate" + source ${venv}/bin/activate - install_deps + ${venv}/bin/python3 -m pip install -U pip pip-tools ${pip_compile} --output-file requirements.txt "${requirements_in}" "${requirements_git}" # consider the git requirements for purposes of resolving deps # Then remove any git+ lines from requirements.txt cp requirements.txt requirements_tmp.txt grep -v "^git+" requirements_tmp.txt > requirements.txt && rm requirements_tmp.txt - ${pip_compile} --output-file requirements_ansible_py3.txt "${requirements_ansible_in}" -} - -generate_requirements_v2() { - venv="./venv2" - virtualenv -p python2 "${venv}" - # shellcheck disable=SC1090 - PS1="" . "${venv}/bin/activate" - - install_deps - - ${pip_compile} --output-file requirements_ansible.txt "${requirements_ansible_in}" -} - -generate_patch() { - a="requirements_ansible_py3.txt" - b="requirements_ansible.txt" - replace='; python_version < "3" #' - - # most elegant/quick solution I could come up for now - out="$(diff --ignore-matching-lines='^#' --unified "${a}" "${b}" | \ - awk -v replace="${replace}" '{ if (/^+\w/){ $2=replace; print;} else print; }' | \ - sed 's/ ;/;/g')" - test -n "${out}" - echo "${out}" } main() { - check_prerequisites - _tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)" trap _cleanup INT TERM EXIT @@ -74,19 +35,11 @@ main() { pip_compile="${pip_compile} --upgrade" fi - cp -vf requirements.txt requirements_ansible.txt "${_tmp}" - cp -vf requirements_ansible.txt "${_tmp}/requirements_ansible_py3.txt" - + cp -vf requirements.txt "${_tmp}" cd "${_tmp}" - generate_requirements_v3 - generate_requirements_v2 + generate_requirements - sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible.txt - sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible_py3.txt - generate_patch | patch -p4 requirements_ansible_py3.txt - - cp -vf requirements_ansible_py3.txt "${requirements_ansible}" cp -vf requirements.txt "${requirements}" _cleanup diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 203f64b64e..62964abafe 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -141,6 +141,7 @@ RUN dnf -y install \ strace \ vim \ nmap-ncat \ + libpq-devel \ nodejs \ nss \ make \ From fd658d44c94e8686e0a73ddcfe9518d19d481def Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Thu, 11 Mar 2021 15:40:43 -0500 Subject: [PATCH 11/13] Note that we need to match python versions. (Some libraries don't have the same deps across python versions.) --- requirements/README.md | 3 +++ requirements/updater.sh | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/requirements/README.md b/requirements/README.md index ab694dfbe3..cfcecbb093 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -13,6 +13,9 @@ then run the script: `./updater.sh` +NOTE: `./updater.sh` uses /usr/bin/python3.6, to match the current python version +(3.6) used to build releases. + #### Upgrading Unpinned Dependency If you require a new version of a dependency that does not have a pinned version diff --git a/requirements/updater.sh b/requirements/updater.sh index d184cf8796..2911aaf008 100755 --- a/requirements/updater.sh +++ b/requirements/updater.sh @@ -14,11 +14,11 @@ _cleanup() { generate_requirements() { venv="`pwd`/venv" echo $venv - /usr/bin/python3 -m venv "${venv}" + /usr/bin/python3.6 -m venv "${venv}" # shellcheck disable=SC1090 source ${venv}/bin/activate - ${venv}/bin/python3 -m pip install -U pip pip-tools + ${venv}/bin/python3.6 -m pip install -U pip pip-tools ${pip_compile} --output-file requirements.txt "${requirements_in}" "${requirements_git}" # consider the git requirements for purposes of resolving deps From 5bec4a51c6b8eca41c84201fb41bf33cc1fe6684 Mon Sep 17 00:00:00 2001 From: Jeff Bradberry Date: Thu, 11 Mar 2021 15:50:57 -0500 Subject: [PATCH 12/13] Undo the polymorphic.SET_NULL for Organization It isn't polymorphic. --- .../0131_undo_org_polymorphic_ee.py | 19 +++++++++++++++++++ awx/main/models/organization.py | 3 +-- 2 files changed, 20 insertions(+), 2 deletions(-) create mode 100644 awx/main/migrations/0131_undo_org_polymorphic_ee.py diff --git a/awx/main/migrations/0131_undo_org_polymorphic_ee.py b/awx/main/migrations/0131_undo_org_polymorphic_ee.py new file mode 100644 index 0000000000..0805992243 --- /dev/null +++ b/awx/main/migrations/0131_undo_org_polymorphic_ee.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2021-03-11 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0130_ee_polymorphic_set_null'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + ] diff --git a/awx/main/models/organization.py b/awx/main/models/organization.py index f087b8eb6b..f0ecfea5c7 100644 --- a/awx/main/models/organization.py +++ b/awx/main/models/organization.py @@ -27,7 +27,6 @@ from awx.main.models.rbac import ( ) from awx.main.models.unified_jobs import UnifiedJob from awx.main.models.mixins import ResourceMixin, CustomVirtualEnvMixin, RelatedJobsMixin -from awx.main.utils import polymorphic __all__ = ['Organization', 'Team', 'Profile', 'UserSessionMembership'] @@ -67,7 +66,7 @@ class Organization(CommonModel, NotificationFieldsModel, ResourceMixin, CustomVi null=True, blank=True, default=None, - on_delete=polymorphic.SET_NULL, + on_delete=models.SET_NULL, related_name='+', help_text=_('The default execution environment for jobs run by this organization.'), ) From 0ee49dae7623fa2ebc1fdfb82612b92dc0245e82 Mon Sep 17 00:00:00 2001 From: Nikhil Jain Date: Fri, 12 Mar 2021 14:57:52 +0530 Subject: [PATCH 13/13] fix tower collection integration test race condition --- .../targets/tower_job_launch/tasks/main.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml b/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml index 14d7f7c236..4599682fbf 100644 --- a/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml +++ b/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml @@ -164,6 +164,18 @@ that: - "result is changed" +- name: Wait for a job template to complete + tower_job_wait: + job_id: "{{ result.id }}" + max_interval: 10 + timeout: 120 + register: result + +- assert: + that: + - "result is not changed" + - "result.status == 'successful'" + - name: Get the job tower_job_list: query: {"id": "{{result.id}}"}