diff --git a/awx/conf/settings.py b/awx/conf/settings.py index 4b18e3d9f6..500b4408f5 100644 --- a/awx/conf/settings.py +++ b/awx/conf/settings.py @@ -352,7 +352,6 @@ class SettingsWrapper(UserSettingsHolder): self.cache.set_many(settings_to_cache, timeout=SETTING_CACHE_TIMEOUT) def _get_local(self, name, validate=True): - self.__clean_on_fork__() self._preload_cache() cache_key = Setting.get_cache_key(name) try: diff --git a/awx/main/access.py b/awx/main/access.py index d2a2aa9f3b..cdb0c0eb53 100644 --- a/awx/main/access.py +++ b/awx/main/access.py @@ -777,6 +777,11 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess): @check_superuser def can_change(self, obj, data): + if data and data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return self.user in obj.admin_role def can_delete(self, obj): @@ -1385,14 +1390,29 @@ class ProjectAccess(NotificationAttachMixin, BaseAccess): def can_add(self, data): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'project_admin_role').exists() - return (self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and - self.check_related('credential', Credential, data, role_field='use_role')) + + if data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + + return ( + self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and + self.check_related('credential', Credential, data, role_field='use_role') + ) @check_superuser def can_change(self, obj, data): - return (self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and - self.user in obj.admin_role and - self.check_related('credential', Credential, data, obj=obj, role_field='use_role')) + if data and data.get('default_environment'): + ee = get_object_from_data('default_environment', ExecutionEnvironment, data, obj=obj) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + + return ( + self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and + self.user in obj.admin_role and + self.check_related('credential', Credential, data, obj=obj, role_field='use_role') + ) @check_superuser def can_start(self, obj, validate_license=True): @@ -1497,6 +1517,10 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.user not in inventory.use_role: return False + ee = get_value(ExecutionEnvironment, 'execution_environment') + if ee and not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + project = get_value(Project, 'project') # If the user has admin access to the project (as an org admin), should # be able to proceed without additional checks. @@ -1544,6 +1568,11 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.changes_are_non_sensitive(obj, data): return True + if data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + for required_field, cls in (('inventory', Inventory), ('project', Project)): is_mandatory = True if not getattr(obj, '{}_id'.format(required_field)): @@ -1974,6 +2003,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess): if not data: # So the browseable API will work return Organization.accessible_objects(self.user, 'workflow_admin_role').exists() + if data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return ( self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True) and self.check_related('inventory', Inventory, data, role_field='use_role') @@ -2023,6 +2057,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess): if self.user.is_superuser: return True + if data and data.get('execution_environment'): + ee = get_object_from_data('execution_environment', ExecutionEnvironment, data) + if not self.user.can_access(ExecutionEnvironment, 'read', ee): + return False + return ( self.check_related('organization', Organization, data, role_field='workflow_admin_role', obj=obj) and self.check_related('inventory', Inventory, data, role_field='use_role', obj=obj) and diff --git a/awx/main/dispatch/worker/task.py b/awx/main/dispatch/worker/task.py index 80a414770b..d71d62686f 100644 --- a/awx/main/dispatch/worker/task.py +++ b/awx/main/dispatch/worker/task.py @@ -6,6 +6,7 @@ import traceback from kubernetes.config import kube_config +from django.conf import settings from django_guid.middleware import GuidMiddleware from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown @@ -85,6 +86,7 @@ class TaskWorker(BaseWorker): 'task': u'awx.main.tasks.RunProjectUpdate' } ''' + settings.__clean_on_fork__() result = None try: result = self.run_callable(body) diff --git a/awx/main/middleware.py b/awx/main/middleware.py index 8bfd273811..2d509c9c61 100644 --- a/awx/main/middleware.py +++ b/awx/main/middleware.py @@ -45,7 +45,10 @@ class TimingMiddleware(threading.local, MiddlewareMixin): response['X-API-Total-Time'] = '%0.3fs' % total_time if settings.AWX_REQUEST_PROFILE: response['X-API-Profile-File'] = self.prof.stop() - perf_logger.info('api response times', extra=dict(python_objects=dict(request=request, response=response))) + perf_logger.info( + f'request: {request}, response_time: {response["X-API-Total-Time"]}', + extra=dict(python_objects=dict(request=request, response=response, X_API_TOTAL_TIME=response["X-API-Total-Time"])) + ) return response diff --git a/awx/main/migrations/0130_ee_polymorphic_set_null.py b/awx/main/migrations/0130_ee_polymorphic_set_null.py new file mode 100644 index 0000000000..a9a0b63715 --- /dev/null +++ b/awx/main/migrations/0130_ee_polymorphic_set_null.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.16 on 2021-03-11 16:25 + +import awx.main.utils.polymorphic +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0129_unifiedjob_installed_collections'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='project', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run using this project.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='unifiedjob', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobs', to='main.ExecutionEnvironment'), + ), + migrations.AlterField( + model_name='unifiedjobtemplate', + name='execution_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobtemplates', to='main.ExecutionEnvironment'), + ), + ] diff --git a/awx/main/migrations/0131_undo_org_polymorphic_ee.py b/awx/main/migrations/0131_undo_org_polymorphic_ee.py new file mode 100644 index 0000000000..0805992243 --- /dev/null +++ b/awx/main/migrations/0131_undo_org_polymorphic_ee.py @@ -0,0 +1,19 @@ +# Generated by Django 2.2.16 on 2021-03-11 20:50 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('main', '0130_ee_polymorphic_set_null'), + ] + + operations = [ + migrations.AlterField( + model_name='organization', + name='default_environment', + field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'), + ), + ] diff --git a/awx/main/models/mixins.py b/awx/main/models/mixins.py index 549c93607d..9df38b3fa4 100644 --- a/awx/main/models/mixins.py +++ b/awx/main/models/mixins.py @@ -22,7 +22,7 @@ from awx.main.models.base import prevent_search from awx.main.models.rbac import ( Role, RoleAncestorEntry, get_roles_on_resource ) -from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser +from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.utils.polymorphic import build_polymorphic_ctypes_map from awx.main.fields import JSONField, AskForField @@ -450,7 +450,7 @@ class ExecutionEnvironmentMixin(models.Model): null=True, blank=True, default=None, - on_delete=models.SET_NULL, + on_delete=polymorphic.SET_NULL, related_name='%(class)ss', help_text=_('The container image to be used for execution.'), ) diff --git a/awx/main/models/projects.py b/awx/main/models/projects.py index c9bf87f408..5a613b6c21 100644 --- a/awx/main/models/projects.py +++ b/awx/main/models/projects.py @@ -35,7 +35,7 @@ from awx.main.models.mixins import ( CustomVirtualEnvMixin, RelatedJobsMixin ) -from awx.main.utils import update_scm_url +from awx.main.utils import update_scm_url, polymorphic from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook from awx.main.fields import ImplicitRoleField from awx.main.models.rbac import ( @@ -272,7 +272,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn null=True, blank=True, default=None, - on_delete=models.SET_NULL, + on_delete=polymorphic.SET_NULL, related_name='+', help_text=_('The default execution environment for jobs run using this project.'), ) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index ba8b8c69cf..502afa5b18 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1063,6 +1063,11 @@ LOGGING = { 'level': 'INFO', 'propagate': False }, + 'awx.analytics.performance': { + 'handlers': ['console', 'file', 'tower_warnings', 'external_logger'], + 'level': 'DEBUG', + 'propagate': False + }, 'awx.analytics.job_lifecycle': { 'handlers': ['console', 'job_lifecycle'], 'level': 'DEBUG', diff --git a/awx_collection/plugins/modules/tower_user.py b/awx_collection/plugins/modules/tower_user.py index 8f32c42666..b57266dbc0 100644 --- a/awx_collection/plugins/modules/tower_user.py +++ b/awx_collection/plugins/modules/tower_user.py @@ -149,19 +149,19 @@ def main(): # Create the data that gets sent for create and update new_fields = {} - if username: + if username is not None: new_fields['username'] = module.get_item_name(existing_item) if existing_item else username - if first_name: + if first_name is not None: new_fields['first_name'] = first_name - if last_name: + if last_name is not None: new_fields['last_name'] = last_name - if email: + if email is not None: new_fields['email'] = email - if is_superuser: + if is_superuser is not None: new_fields['is_superuser'] = is_superuser - if is_system_auditor: + if is_system_auditor is not None: new_fields['is_system_auditor'] = is_system_auditor - if password: + if password is not None: new_fields['password'] = password # If the state was present and we can let the module build or update the existing item, this will return on its own diff --git a/awx_collection/test/awx/test_user.py b/awx_collection/test/awx/test_user.py index 6c49f04121..6a0dfa123d 100644 --- a/awx_collection/test/awx/test_user.py +++ b/awx_collection/test/awx/test_user.py @@ -57,3 +57,23 @@ def test_update_password_on_create(run_module, admin_user, mock_auth_stuff): assert not result.get('failed', False), result.get('msg', result) assert not result.get('changed') + + +@pytest.mark.django_db +def test_update_user(run_module, admin_user, mock_auth_stuff): + result = run_module('tower_user', dict( + username='Bob', + password='pass4word', + is_system_auditor=True + ), admin_user) + assert not result.get('failed', False), result.get('msg', result) + assert result.get('changed'), result + + update_result = run_module('tower_user', dict( + username='Bob', + is_system_auditor=False + ), admin_user) + + assert update_result.get('changed') + user = User.objects.get(id=result['id']) + assert not user.is_system_auditor diff --git a/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml b/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml index e789a2bc7b..4599682fbf 100644 --- a/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml +++ b/awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml @@ -164,10 +164,17 @@ that: - "result is changed" -- name: Wait for 2nd job max 120s +- name: Wait for a job template to complete tower_job_wait: - job_id: "{{result.id}}" + job_id: "{{ result.id }}" + max_interval: 10 timeout: 120 + register: result + +- assert: + that: + - "result is not changed" + - "result.status == 'successful'" - name: Get the job tower_job_list: diff --git a/awxkit/setup.py b/awxkit/setup.py index 6e357a800f..b446ba0a90 100644 --- a/awxkit/setup.py +++ b/awxkit/setup.py @@ -70,7 +70,7 @@ setup( python_requires=">=3.6", extras_require={ 'formatting': ['jq'], - 'websockets': ['websocket-client>0.54.0'], + 'websockets': ['websocket-client==0.57.0'], 'crypto': ['cryptography'] }, license='Apache 2.0', diff --git a/docs/licenses/click.txt b/docs/licenses/click.txt new file mode 100644 index 0000000000..d12a849186 --- /dev/null +++ b/docs/licenses/click.txt @@ -0,0 +1,28 @@ +Copyright 2014 Pallets + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/requirements/README.md b/requirements/README.md index 22dfe8d62d..cfcecbb093 100644 --- a/requirements/README.md +++ b/requirements/README.md @@ -1,15 +1,11 @@ # Dependency Management -The `requirements.txt` and `requirements_ansible.txt` files are generated from `requirements.in` and `requirements_ansible.in`, respectively, using `pip-tools` `pip-compile`. +The `requirements.txt` file is generated from `requirements.in`, using `pip-tools` `pip-compile`. ## How To Use Commands should be run from inside the `./requirements` directory of the awx repository. -Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed. The development container image should have all these. - -Even in the dev container, you may still have to dnf install `libpq-devel libcurl-devel`. - ### Upgrading or Adding Select Libraries If you need to add or upgrade one targeted library, then modify `requirements.in`, @@ -17,6 +13,9 @@ then run the script: `./updater.sh` +NOTE: `./updater.sh` uses /usr/bin/python3.6, to match the current python version +(3.6) used to build releases. + #### Upgrading Unpinned Dependency If you require a new version of a dependency that does not have a pinned version @@ -33,14 +32,6 @@ You can upgrade (`pip-compile --upgrade`) the dependencies by running `./updater.sh upgrade`. -## What The Script Does - -This script will: - - - Update `requirements.txt` based on `requirements.in` - - Update/generate `requirements_ansible.txt` based on `requirements_ansible.in` - - including an automated patch that adds `python_version < "3"` for Python 2 backward compatibility - ## Licenses and Source Files If any library has a change to its license with the upgrade, then the license for that library @@ -129,11 +120,6 @@ This breaks a very large amount of AWX code that assumes these fields are returned as dicts. Upgrading this library will require a refactor to accomidate this change. -### wheel - -azure-cli-core requires a version of wheel that is incompatible with -certain packages building with later versions of pip, so we override it. - ### pip and setuptools The offline installer needs to have functionality confirmed before upgrading these. diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 6256afd061..ba3cfe3087 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,135 +1,430 @@ -adal==1.2.2 # via msrestazure -aiohttp==3.6.2 # via -r /awx_devel/requirements/requirements.in -aioredis==1.3.1 # via channels-redis -# ansible-runner==1.4.7 # via -r /awx_devel/requirements/requirements.in -ansiconv==1.0.0 # via -r /awx_devel/requirements/requirements.in -asciichartpy==1.5.25 # via -r /awx_devel/requirements/requirements.in -asgiref==3.2.5 # via channels, channels-redis, daphne -async-timeout==3.0.1 # via aiohttp, aioredis -attrs==19.3.0 # via aiohttp, automat, jsonschema, service-identity, twisted -autobahn==20.12.3 # via -r /awx_devel/requirements/requirements.in, daphne -automat==20.2.0 # via twisted -azure-common==1.1.25 # via azure-keyvault -azure-keyvault==1.1.0 # via -r /awx_devel/requirements/requirements.in -azure-nspkg==3.0.2 # via azure-keyvault -cachetools==4.0.0 # via google-auth -# don't add certifi to our requirements, because we have our own specific implementation: https://github.com/ansible/system-certifi -cffi==1.14.0 # via cryptography -channels-redis==3.1.0 # via -r /awx_devel/requirements/requirements.in -channels==2.4.0 # via -r /awx_devel/requirements/requirements.in, channels-redis -chardet==3.0.4 # via aiohttp, requests -constantly==15.1.0 # via twisted -cryptography==2.9.2 # via -r /awx_devel/requirements/requirements.in, adal, autobahn, azure-keyvault, pyopenssl, service-identity, social-auth-core -daphne==2.4.1 # via -r /awx_devel/requirements/requirements.in, channels -defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core -dictdiffer==0.8.1 # via openshift -django-auth-ldap==2.1.0 # via -r /awx_devel/requirements/requirements.in -django-cors-headers==3.2.1 # via -r /awx_devel/requirements/requirements.in -django-crum==0.7.5 # via -r /awx_devel/requirements/requirements.in -django-extensions==2.2.9 # via -r /awx_devel/requirements/requirements.in -django-guid==2.2.0 # via -r /awx_devel/requirements/requirements.in -django-jsonfield==1.2.0 # via -r /awx_devel/requirements/requirements.in -django-oauth-toolkit==1.1.3 # via -r /awx_devel/requirements/requirements.in -django-pglocks==1.0.4 # via -r /awx_devel/requirements/requirements.in -django-polymorphic==2.1.2 # via -r /awx_devel/requirements/requirements.in -django-qsstats-magic==1.1.0 # via -r /awx_devel/requirements/requirements.in -django-radius==1.3.3 # via -r /awx_devel/requirements/requirements.in -django-redis==4.5.0 # via -r /awx_devel/requirements/requirements.in -django-solo==1.1.3 # via -r /awx_devel/requirements/requirements.in -django-split-settings==1.0.0 # via -r /awx_devel/requirements/requirements.in -django-taggit==1.2.0 # via -r /awx_devel/requirements/requirements.in -django==2.2.16 # via -r /awx_devel/requirements/requirements.in, channels, django-auth-ldap, django-cors-headers, django-crum, django-guid, django-jsonfield, django-oauth-toolkit, django-polymorphic, django-taggit, djangorestframework -djangorestframework-yaml==1.0.3 # via -r /awx_devel/requirements/requirements.in -djangorestframework==3.12.1 # via -r /awx_devel/requirements/requirements.in -docutils==0.16 # via python-daemon -future==0.16.0 # via django-radius -gitdb==4.0.2 # via gitpython -gitpython==3.1.7 # via -r /awx_devel/requirements/requirements.in -google-auth==1.11.3 # via kubernetes -hiredis==1.0.1 # via aioredis -hyperlink==20.0.1 # via autobahn, twisted -idna-ssl==1.1.0 # via aiohttp -idna==2.9 # via hyperlink, idna-ssl, requests, twisted, yarl -importlib-metadata==1.5.0 # via importlib-resources, irc, jsonschema -importlib-resources==1.4.0 # via jaraco.text -incremental==17.5.0 # via twisted -irc==18.0.0 # via -r /awx_devel/requirements/requirements.in -isodate==0.6.0 # via msrest, python3-saml -jaraco.classes==3.1.0 # via jaraco.collections -jaraco.collections==3.0.0 # via irc -jaraco.functools==3.0.0 # via irc, jaraco.text, tempora -jaraco.logging==3.0.0 # via irc -jaraco.stream==3.0.0 # via irc -jaraco.text==3.2.0 # via irc, jaraco.collections -jinja2==2.11.2 # via -r /awx_devel/requirements/requirements.in, openshift -json-log-formatter==0.3.0 # via -r /awx_devel/requirements/requirements.in -jsonschema==3.2.0 # via -r /awx_devel/requirements/requirements.in -kubernetes==11.0.0 # via openshift -lockfile==0.12.2 # via python-daemon -lxml==4.5.0 # via xmlsec -markdown==3.2.1 # via -r /awx_devel/requirements/requirements.in -markupsafe==1.1.1 # via jinja2 -more-itertools==8.2.0 # via irc, jaraco.classes, jaraco.functools -msgpack==1.0.0 # via channels-redis -msrest==0.6.11 # via azure-keyvault, msrestazure -msrestazure==0.6.3 # via azure-keyvault -multidict==4.7.5 # via aiohttp, yarl -netaddr==0.7.19 # via pyrad -oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core -openshift==0.11.0 # via -r /awx_devel/requirements/requirements.in -pexpect==4.7.0 # via -r /awx_devel/requirements/requirements.in, ansible-runner -pkgconfig==1.5.1 # via xmlsec -prometheus-client==0.7.1 # via -r /awx_devel/requirements/requirements.in -psutil==5.7.0 # via ansible-runner -psycopg2==2.8.4 # via -r /awx_devel/requirements/requirements.in -ptyprocess==0.6.0 # via pexpect -pyasn1-modules==0.2.8 # via google-auth, python-ldap, service-identity -pyasn1==0.4.8 # via pyasn1-modules, python-ldap, rsa, service-identity -pycparser==2.20 # via cffi -pygerduty==0.38.2 # via -r /awx_devel/requirements/requirements.in -pyhamcrest==2.0.2 # via twisted -pyjwt==1.7.1 # via adal, social-auth-core, twilio -pyopenssl==19.1.0 # via twisted -pyparsing==2.4.6 # via -r /awx_devel/requirements/requirements.in -pyrad==2.3 # via django-radius -pyrsistent==0.15.7 # via jsonschema -python-daemon==2.2.4 # via ansible-runner -python-dateutil==2.8.1 # via adal, kubernetes -python-ldap==3.3.1 # via -r /awx_devel/requirements/requirements.in, django-auth-ldap -python-string-utils==1.0.0 # via openshift -python3-openid==3.1.0 # via social-auth-core -python3-saml==1.9.0 # via -r /awx_devel/requirements/requirements.in -pytz==2019.3 # via django, irc, tempora, twilio -pyyaml==5.4.1 # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes -redis==3.4.1 # via -r /awx_devel/requirements/requirements.in, django-redis -requests-oauthlib==1.3.0 # via kubernetes, msrest, social-auth-core -requests==2.23.0 # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-oauthlib, slackclient, social-auth-core, twilio -rsa==4.0 # via google-auth -ruamel.yaml.clib==0.2.0 # via ruamel.yaml -ruamel.yaml==0.16.10 # via openshift -schedule==0.6.0 # via -r /awx_devel/requirements/requirements.in -service-identity==18.1.0 # via twisted -six==1.14.0 # via ansible-runner, automat, cryptography, django-extensions, django-pglocks, google-auth, isodate, jaraco.collections, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyopenssl, pyrad, pyrsistent, python-dateutil, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, twilio, websocket-client -slackclient==1.1.2 # via -r /awx_devel/requirements/requirements.in -smmap==3.0.1 # via gitdb -social-auth-app-django==3.1.0 # via -r /awx_devel/requirements/requirements.in -social-auth-core==3.3.1 # via -r /awx_devel/requirements/requirements.in, social-auth-app-django -sqlparse==0.3.1 # via django -tacacs_plus==1.0 # via -r /awx_devel/requirements/requirements.in -tempora==2.1.0 # via irc, jaraco.logging -twilio==6.37.0 # via -r /awx_devel/requirements/requirements.in -twisted[tls]==20.3.0 # via -r /awx_devel/requirements/requirements.in, daphne -txaio==20.12.1 # via autobahn -typing-extensions==3.7.4.1 # via aiohttp -urllib3==1.25.8 # via kubernetes, requests -uwsgi==2.0.18 # via -r /awx_devel/requirements/requirements.in -uwsgitop==0.11 # via -r /awx_devel/requirements/requirements.in -websocket-client==0.57.0 # via kubernetes, slackclient -xmlsec==1.3.3 # via python3-saml -yarl==1.4.2 # via aiohttp -zipp==3.1.0 # via importlib-metadata, importlib-resources -zope.interface==5.0.0 # via twisted +adal==1.2.2 + # via msrestazure +aiohttp==3.6.2 + # via -r /awx_devel/requirements/requirements.in +aioredis==1.3.1 + # via channels-redis +#ansible-runner==1.4.7 + # via + # -r /awx_devel/requirements/requirements.in + # -r /awx_devel/requirements/requirements_git.txt +ansiconv==1.0.0 + # via -r /awx_devel/requirements/requirements.in +asciichartpy==1.5.25 + # via -r /awx_devel/requirements/requirements.in +asgiref==3.2.5 + # via + # channels + # channels-redis + # daphne +async-timeout==3.0.1 + # via + # aiohttp + # aioredis +attrs==19.3.0 + # via + # aiohttp + # automat + # jsonschema + # service-identity + # twisted +autobahn==20.12.3 + # via + # -r /awx_devel/requirements/requirements.in + # daphne +automat==20.2.0 + # via twisted +azure-common==1.1.25 + # via azure-keyvault +azure-keyvault==1.1.0 + # via -r /awx_devel/requirements/requirements.in +azure-nspkg==3.0.2 + # via azure-keyvault +cachetools==4.0.0 + # via google-auth + # via + # -r /awx_devel/requirements/requirements_git.txt + # kubernetes + # msrest + # requests +cffi==1.14.0 + # via cryptography +channels-redis==3.1.0 + # via -r /awx_devel/requirements/requirements.in +channels==2.4.0 + # via + # -r /awx_devel/requirements/requirements.in + # channels-redis +chardet==3.0.4 + # via + # aiohttp + # requests +click==7.1.2 + # via receptorctl +constantly==15.1.0 + # via twisted +cryptography==2.9.2 + # via + # -r /awx_devel/requirements/requirements.in + # adal + # autobahn + # azure-keyvault + # pyopenssl + # service-identity + # social-auth-core +daphne==2.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # channels +defusedxml==0.6.0 + # via + # python3-openid + # python3-saml + # social-auth-core +dictdiffer==0.8.1 + # via openshift +django-auth-ldap==2.1.0 + # via -r /awx_devel/requirements/requirements.in +django-cors-headers==3.2.1 + # via -r /awx_devel/requirements/requirements.in +django-crum==0.7.5 + # via -r /awx_devel/requirements/requirements.in +django-extensions==2.2.9 + # via -r /awx_devel/requirements/requirements.in +django-guid==2.2.0 + # via -r /awx_devel/requirements/requirements.in +django-jsonfield==1.2.0 + # via -r /awx_devel/requirements/requirements.in +django-oauth-toolkit==1.1.3 + # via -r /awx_devel/requirements/requirements.in +django-pglocks==1.0.4 + # via -r /awx_devel/requirements/requirements.in +django-polymorphic==2.1.2 + # via -r /awx_devel/requirements/requirements.in +django-qsstats-magic==1.1.0 + # via -r /awx_devel/requirements/requirements.in +django-radius==1.3.3 + # via -r /awx_devel/requirements/requirements.in +django-redis==4.5.0 + # via -r /awx_devel/requirements/requirements.in +django-solo==1.1.3 + # via -r /awx_devel/requirements/requirements.in +django-split-settings==1.0.0 + # via -r /awx_devel/requirements/requirements.in +django-taggit==1.2.0 + # via -r /awx_devel/requirements/requirements.in +django==2.2.16 + # via + # -r /awx_devel/requirements/requirements.in + # channels + # django-auth-ldap + # django-cors-headers + # django-crum + # django-guid + # django-jsonfield + # django-oauth-toolkit + # django-polymorphic + # django-taggit + # djangorestframework +djangorestframework-yaml==1.0.3 + # via -r /awx_devel/requirements/requirements.in +djangorestframework==3.12.1 + # via -r /awx_devel/requirements/requirements.in +docutils==0.16 + # via python-daemon +future==0.16.0 + # via django-radius +gitdb==4.0.2 + # via gitpython +gitpython==3.1.7 + # via -r /awx_devel/requirements/requirements.in +google-auth==1.11.3 + # via kubernetes +hiredis==1.0.1 + # via aioredis +hyperlink==20.0.1 + # via + # autobahn + # twisted +idna-ssl==1.1.0 + # via aiohttp +idna==2.9 + # via + # hyperlink + # idna-ssl + # requests + # twisted + # yarl +importlib-metadata==1.5.0 + # via + # importlib-resources + # irc + # jsonschema +importlib-resources==1.4.0 + # via jaraco.text +incremental==17.5.0 + # via twisted +irc==18.0.0 + # via -r /awx_devel/requirements/requirements.in +isodate==0.6.0 + # via + # msrest + # python3-saml +jaraco.classes==3.1.0 + # via jaraco.collections +jaraco.collections==3.0.0 + # via irc +jaraco.functools==3.0.0 + # via + # irc + # jaraco.text + # tempora +jaraco.logging==3.0.0 + # via irc +jaraco.stream==3.0.0 + # via irc +jaraco.text==3.2.0 + # via + # irc + # jaraco.collections +jinja2==2.11.2 + # via + # -r /awx_devel/requirements/requirements.in + # openshift +json-log-formatter==0.3.0 + # via -r /awx_devel/requirements/requirements.in +jsonschema==3.2.0 + # via -r /awx_devel/requirements/requirements.in +kubernetes==11.0.0 + # via openshift +lockfile==0.12.2 + # via python-daemon +lxml==4.5.0 + # via xmlsec +markdown==3.2.1 + # via -r /awx_devel/requirements/requirements.in +markupsafe==1.1.1 + # via jinja2 +more-itertools==8.2.0 + # via + # irc + # jaraco.classes + # jaraco.functools +msgpack==1.0.0 + # via channels-redis +msrest==0.6.11 + # via + # azure-keyvault + # msrestazure +msrestazure==0.6.3 + # via azure-keyvault +multidict==4.7.5 + # via + # aiohttp + # yarl +netaddr==0.7.19 + # via pyrad +oauthlib==3.1.0 + # via + # django-oauth-toolkit + # requests-oauthlib + # social-auth-core +openshift==0.11.0 + # via -r /awx_devel/requirements/requirements.in +pexpect==4.7.0 + # via + # -r /awx_devel/requirements/requirements.in + # ansible-runner +pkgconfig==1.5.1 + # via xmlsec +prometheus-client==0.7.1 + # via -r /awx_devel/requirements/requirements.in +psutil==5.7.0 + # via ansible-runner +psycopg2==2.8.4 + # via -r /awx_devel/requirements/requirements.in +ptyprocess==0.6.0 + # via pexpect +pyasn1-modules==0.2.8 + # via + # google-auth + # python-ldap + # service-identity +pyasn1==0.4.8 + # via + # pyasn1-modules + # python-ldap + # rsa + # service-identity +pycparser==2.20 + # via cffi +pygerduty==0.38.2 + # via -r /awx_devel/requirements/requirements.in +pyhamcrest==2.0.2 + # via twisted +pyjwt==1.7.1 + # via + # adal + # social-auth-core + # twilio +pyopenssl==19.1.0 + # via twisted +pyparsing==2.4.6 + # via -r /awx_devel/requirements/requirements.in +pyrad==2.3 + # via django-radius +pyrsistent==0.15.7 + # via jsonschema +python-daemon==2.2.4 + # via ansible-runner +python-dateutil==2.8.1 + # via + # adal + # kubernetes + # receptorctl +python-ldap==3.3.1 + # via + # -r /awx_devel/requirements/requirements.in + # django-auth-ldap +python-string-utils==1.0.0 + # via openshift +python3-openid==3.1.0 + # via social-auth-core +python3-saml==1.9.0 + # via -r /awx_devel/requirements/requirements.in +pytz==2019.3 + # via + # django + # irc + # tempora + # twilio +pyyaml==5.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # ansible-runner + # djangorestframework-yaml + # kubernetes + # receptorctl + # via -r /awx_devel/requirements/requirements_git.txt +redis==3.4.1 + # via + # -r /awx_devel/requirements/requirements.in + # django-redis +requests-oauthlib==1.3.0 + # via + # kubernetes + # msrest + # social-auth-core +requests==2.23.0 + # via + # -r /awx_devel/requirements/requirements.in + # adal + # azure-keyvault + # django-oauth-toolkit + # kubernetes + # msrest + # requests-oauthlib + # slackclient + # social-auth-core + # twilio +rsa==4.0 + # via google-auth +ruamel.yaml.clib==0.2.0 + # via ruamel.yaml +ruamel.yaml==0.16.10 + # via openshift +schedule==0.6.0 + # via -r /awx_devel/requirements/requirements.in +service-identity==18.1.0 + # via twisted +six==1.14.0 + # via + # ansible-runner + # automat + # cryptography + # django-extensions + # django-pglocks + # google-auth + # isodate + # jaraco.collections + # jaraco.logging + # jaraco.text + # jsonschema + # kubernetes + # openshift + # pygerduty + # pyopenssl + # pyrad + # pyrsistent + # python-dateutil + # slackclient + # social-auth-app-django + # social-auth-core + # tacacs-plus + # twilio + # websocket-client +slackclient==1.1.2 + # via -r /awx_devel/requirements/requirements.in +smmap==3.0.1 + # via gitdb +social-auth-app-django==3.1.0 + # via -r /awx_devel/requirements/requirements.in +social-auth-core==3.3.1 + # via + # -r /awx_devel/requirements/requirements.in + # social-auth-app-django +sqlparse==0.3.1 + # via django +tacacs_plus==1.0 + # via -r /awx_devel/requirements/requirements.in +tempora==2.1.0 + # via + # irc + # jaraco.logging +twilio==6.37.0 + # via -r /awx_devel/requirements/requirements.in +twisted[tls]==20.3.0 + # via + # -r /awx_devel/requirements/requirements.in + # daphne +txaio==20.12.1 + # via autobahn +typing-extensions==3.7.4.1 + # via aiohttp +urllib3==1.25.8 + # via + # kubernetes + # requests +uwsgi==2.0.18 + # via -r /awx_devel/requirements/requirements.in +uwsgitop==0.11 + # via -r /awx_devel/requirements/requirements.in +websocket-client==0.57.0 + # via + # kubernetes + # slackclient +xmlsec==1.3.3 + # via python3-saml +yarl==1.4.2 + # via aiohttp +zipp==3.1.0 + # via + # importlib-metadata + # importlib-resources +zope.interface==5.0.0 + # via twisted # The following packages are considered to be unsafe in a requirements file: -pip==19.3.1 # via -r /awx_devel/requirements/requirements.in -setuptools==41.6.0 # via -r /awx_devel/requirements/requirements.in, asciichartpy, google-auth, jsonschema, kubernetes, markdown, python-daemon, zope.interface +pip==19.3.1 + # via -r /awx_devel/requirements/requirements.in +setuptools==41.6.0 + # via + # -r /awx_devel/requirements/requirements.in + # asciichartpy + # google-auth + # jsonschema + # kubernetes + # markdown + # python-daemon + # receptorctl + # zope.interface diff --git a/requirements/requirements_git.txt b/requirements/requirements_git.txt index 74f1fb4a9e..f2f3abaa7a 100644 --- a/requirements/requirements_git.txt +++ b/requirements/requirements_git.txt @@ -1,3 +1,3 @@ git+https://github.com/ansible/system-certifi.git@devel#egg=certifi git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner -git+https://github.com/project-receptor/receptor.git@#egg=receptorctl&subdirectory=receptorctl +git+https://github.com/project-receptor/receptor.git@0.9.6#egg=receptorctl&subdirectory=receptorctl diff --git a/requirements/updater.sh b/requirements/updater.sh index c58f1a0f62..2911aaf008 100755 --- a/requirements/updater.sh +++ b/requirements/updater.sh @@ -2,71 +2,32 @@ set -ue requirements_in="$(readlink -f ./requirements.in)" -requirements_ansible_in="$(readlink -f ./requirements_ansible.in)" requirements="$(readlink -f ./requirements.txt)" requirements_git="$(readlink -f ./requirements_git.txt)" -requirements_ansible="$(readlink -f ./requirements_ansible.txt)" pip_compile="pip-compile --no-header --quiet -r --allow-unsafe" -check_prerequisites() { - for thing in patch awk python3 python2 virtualenv ; do - command -v $thing >/dev/null 2>&1 || { echo "$thing not installed or available. Please fix this before running." ; exit 1 ; } - done -} - _cleanup() { cd / test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}" } -install_deps() { - pip install pip --upgrade - pip install "pip-tools==5.4.0" # see https://github.com/jazzband/pip-tools/pull/1237 -} - -generate_requirements_v3() { - venv="./venv3" - python3 -m venv "${venv}" +generate_requirements() { + venv="`pwd`/venv" + echo $venv + /usr/bin/python3.6 -m venv "${venv}" # shellcheck disable=SC1090 - . "${venv}/bin/activate" + source ${venv}/bin/activate - install_deps + ${venv}/bin/python3.6 -m pip install -U pip pip-tools ${pip_compile} --output-file requirements.txt "${requirements_in}" "${requirements_git}" # consider the git requirements for purposes of resolving deps # Then remove any git+ lines from requirements.txt cp requirements.txt requirements_tmp.txt grep -v "^git+" requirements_tmp.txt > requirements.txt && rm requirements_tmp.txt - ${pip_compile} --output-file requirements_ansible_py3.txt "${requirements_ansible_in}" -} - -generate_requirements_v2() { - venv="./venv2" - virtualenv -p python2 "${venv}" - # shellcheck disable=SC1090 - PS1="" . "${venv}/bin/activate" - - install_deps - - ${pip_compile} --output-file requirements_ansible.txt "${requirements_ansible_in}" -} - -generate_patch() { - a="requirements_ansible_py3.txt" - b="requirements_ansible.txt" - replace='; python_version < "3" #' - - # most elegant/quick solution I could come up for now - out="$(diff --ignore-matching-lines='^#' --unified "${a}" "${b}" | \ - awk -v replace="${replace}" '{ if (/^+\w/){ $2=replace; print;} else print; }' | \ - sed 's/ ;/;/g')" - test -n "${out}" - echo "${out}" } main() { - check_prerequisites - _tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)" trap _cleanup INT TERM EXIT @@ -74,19 +35,11 @@ main() { pip_compile="${pip_compile} --upgrade" fi - cp -vf requirements.txt requirements_ansible.txt "${_tmp}" - cp -vf requirements_ansible.txt "${_tmp}/requirements_ansible_py3.txt" - + cp -vf requirements.txt "${_tmp}" cd "${_tmp}" - generate_requirements_v3 - generate_requirements_v2 + generate_requirements - sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible.txt - sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible_py3.txt - generate_patch | patch -p4 requirements_ansible_py3.txt - - cp -vf requirements_ansible_py3.txt "${requirements_ansible}" cp -vf requirements.txt "${requirements}" _cleanup diff --git a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 index 203f64b64e..62964abafe 100644 --- a/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 +++ b/tools/ansible/roles/dockerfile/templates/Dockerfile.j2 @@ -141,6 +141,7 @@ RUN dnf -y install \ strace \ vim \ nmap-ncat \ + libpq-devel \ nodejs \ nss \ make \