External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-03-14 23:47:28 -02:30
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'devel' into copy_awx_collection

Browse Source
This commit is contained in:
Sean Sullivan
2021-03-12 15:26:59 -06:00
committed by GitHub
parent 10c357d0f1 9c8dbdc7a5
commit 0f04851ab7
19 changed files with 619 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
awx/conf/settings.py
View File

@@ -352,7 +352,6 @@ class SettingsWrapper(UserSettingsHolder):
self.cache.set_many(settings_to_cache, timeout=SETTING_CACHE_TIMEOUT) self.cache.set_many(settings_to_cache, timeout=SETTING_CACHE_TIMEOUT)
def _get_local(self, name, validate=True): def _get_local(self, name, validate=True):
self.__clean_on_fork__()
self._preload_cache() self._preload_cache()
cache_key = Setting.get_cache_key(name) cache_key = Setting.get_cache_key(name)
try: try:

49
awx/main/access.py
View File

@@ -777,6 +777,11 @@ class OrganizationAccess(NotificationAttachMixin, BaseAccess):
@check_superuser @check_superuser
def can_change(self, obj, data): def can_change(self, obj, data):
if data and data.get('default_environment'):
ee = get_object_from_data('default_environment', ExecutionEnvironment, data)
if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
return self.user in obj.admin_role return self.user in obj.admin_role
def can_delete(self, obj): def can_delete(self, obj):
@@ -1385,14 +1390,29 @@ class ProjectAccess(NotificationAttachMixin, BaseAccess):
def can_add(self, data): def can_add(self, data):
if not data: # So the browseable API will work if not data: # So the browseable API will work
return Organization.accessible_objects(self.user, 'project_admin_role').exists() return Organization.accessible_objects(self.user, 'project_admin_role').exists()
return (self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and
self.check_related('credential', Credential, data, role_field='use_role')) if data.get('default_environment'):
ee = get_object_from_data('default_environment', ExecutionEnvironment, data)
if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
return (
self.check_related('organization', Organization, data, role_field='project_admin_role', mandatory=True) and
self.check_related('credential', Credential, data, role_field='use_role')
)
@check_superuser @check_superuser
def can_change(self, obj, data): def can_change(self, obj, data):
return (self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and if data and data.get('default_environment'):
self.user in obj.admin_role and ee = get_object_from_data('default_environment', ExecutionEnvironment, data, obj=obj)
self.check_related('credential', Credential, data, obj=obj, role_field='use_role')) if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
return (
self.check_related('organization', Organization, data, obj=obj, role_field='project_admin_role') and
self.user in obj.admin_role and
self.check_related('credential', Credential, data, obj=obj, role_field='use_role')
)
@check_superuser @check_superuser
def can_start(self, obj, validate_license=True): def can_start(self, obj, validate_license=True):
@@ -1497,6 +1517,10 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
if self.user not in inventory.use_role: if self.user not in inventory.use_role:
return False return False
ee = get_value(ExecutionEnvironment, 'execution_environment')
if ee and not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
project = get_value(Project, 'project') project = get_value(Project, 'project')
# If the user has admin access to the project (as an org admin), should # If the user has admin access to the project (as an org admin), should
# be able to proceed without additional checks. # be able to proceed without additional checks.
@@ -1544,6 +1568,11 @@ class JobTemplateAccess(NotificationAttachMixin, BaseAccess):
if self.changes_are_non_sensitive(obj, data): if self.changes_are_non_sensitive(obj, data):
return True return True
if data.get('execution_environment'):
ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
for required_field, cls in (('inventory', Inventory), ('project', Project)): for required_field, cls in (('inventory', Inventory), ('project', Project)):
is_mandatory = True is_mandatory = True
if not getattr(obj, '{}_id'.format(required_field)): if not getattr(obj, '{}_id'.format(required_field)):
@@ -1974,6 +2003,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
if not data: # So the browseable API will work if not data: # So the browseable API will work
return Organization.accessible_objects(self.user, 'workflow_admin_role').exists() return Organization.accessible_objects(self.user, 'workflow_admin_role').exists()
if data.get('execution_environment'):
ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
return ( return (
self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True) and self.check_related('organization', Organization, data, role_field='workflow_admin_role', mandatory=True) and
self.check_related('inventory', Inventory, data, role_field='use_role') self.check_related('inventory', Inventory, data, role_field='use_role')
@@ -2023,6 +2057,11 @@ class WorkflowJobTemplateAccess(NotificationAttachMixin, BaseAccess):
if self.user.is_superuser: if self.user.is_superuser:
return True return True
if data and data.get('execution_environment'):
ee = get_object_from_data('execution_environment', ExecutionEnvironment, data)
if not self.user.can_access(ExecutionEnvironment, 'read', ee):
return False
return ( return (
self.check_related('organization', Organization, data, role_field='workflow_admin_role', obj=obj) and self.check_related('organization', Organization, data, role_field='workflow_admin_role', obj=obj) and
self.check_related('inventory', Inventory, data, role_field='use_role', obj=obj) and self.check_related('inventory', Inventory, data, role_field='use_role', obj=obj) and

2
awx/main/dispatch/worker/task.py
View File

@@ -6,6 +6,7 @@ import traceback
from kubernetes.config import kube_config from kubernetes.config import kube_config
from django.conf import settings
from django_guid.middleware import GuidMiddleware from django_guid.middleware import GuidMiddleware
from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown from awx.main.tasks import dispatch_startup, inform_cluster_of_shutdown
@@ -85,6 +86,7 @@ class TaskWorker(BaseWorker):
'task': u'awx.main.tasks.RunProjectUpdate' 'task': u'awx.main.tasks.RunProjectUpdate'
} }
''' '''
settings.__clean_on_fork__()
result = None result = None
try: try:
result = self.run_callable(body) result = self.run_callable(body)

5
awx/main/middleware.py
View File

@@ -45,7 +45,10 @@ class TimingMiddleware(threading.local, MiddlewareMixin):
response['X-API-Total-Time'] = '%0.3fs' % total_time response['X-API-Total-Time'] = '%0.3fs' % total_time
if settings.AWX_REQUEST_PROFILE: if settings.AWX_REQUEST_PROFILE:
response['X-API-Profile-File'] = self.prof.stop() response['X-API-Profile-File'] = self.prof.stop()
perf_logger.info('api response times', extra=dict(python_objects=dict(request=request, response=response))) perf_logger.info(
f'request: {request}, response_time: {response["X-API-Total-Time"]}',
extra=dict(python_objects=dict(request=request, response=response, X_API_TOTAL_TIME=response["X-API-Total-Time"]))
)
return response return response

34
awx/main/migrations/0130_ee_polymorphic_set_null.py Normal file
View File

@@ -0,0 +1,34 @@
# Generated by Django 2.2.16 on 2021-03-11 16:25
import awx.main.utils.polymorphic
from django.db import migrations, models
class Migration(migrations.Migration):
dependencies = [
('main', '0129_unifiedjob_installed_collections'),
]
operations = [
migrations.AlterField(
model_name='organization',
name='default_environment',
field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'),
),
migrations.AlterField(
model_name='project',
name='default_environment',
field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run using this project.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='+', to='main.ExecutionEnvironment'),
),
migrations.AlterField(
model_name='unifiedjob',
name='execution_environment',
field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobs', to='main.ExecutionEnvironment'),
),
migrations.AlterField(
model_name='unifiedjobtemplate',
name='execution_environment',
field=models.ForeignKey(blank=True, default=None, help_text='The container image to be used for execution.', null=True, on_delete=awx.main.utils.polymorphic.SET_NULL, related_name='unifiedjobtemplates', to='main.ExecutionEnvironment'),
),
]

19
awx/main/migrations/0131_undo_org_polymorphic_ee.py Normal file
View File

@@ -0,0 +1,19 @@
# Generated by Django 2.2.16 on 2021-03-11 20:50
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('main', '0130_ee_polymorphic_set_null'),
]
operations = [
migrations.AlterField(
model_name='organization',
name='default_environment',
field=models.ForeignKey(blank=True, default=None, help_text='The default execution environment for jobs run by this organization.', null=True, on_delete=django.db.models.deletion.SET_NULL, related_name='+', to='main.ExecutionEnvironment'),
),
]

4
awx/main/models/mixins.py
View File

@@ -22,7 +22,7 @@ from awx.main.models.base import prevent_search
from awx.main.models.rbac import ( from awx.main.models.rbac import (
Role, RoleAncestorEntry, get_roles_on_resource Role, RoleAncestorEntry, get_roles_on_resource
) )
from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser from awx.main.utils import parse_yaml_or_json, get_custom_venv_choices, get_licenser, polymorphic
from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted from awx.main.utils.encryption import decrypt_value, get_encryption_key, is_encrypted
from awx.main.utils.polymorphic import build_polymorphic_ctypes_map from awx.main.utils.polymorphic import build_polymorphic_ctypes_map
from awx.main.fields import JSONField, AskForField from awx.main.fields import JSONField, AskForField
@@ -450,7 +450,7 @@ class ExecutionEnvironmentMixin(models.Model):
null=True, null=True,
blank=True, blank=True,
default=None, default=None,
on_delete=models.SET_NULL, on_delete=polymorphic.SET_NULL,
related_name='%(class)ss', related_name='%(class)ss',
help_text=_('The container image to be used for execution.'), help_text=_('The container image to be used for execution.'),
) )

4
awx/main/models/projects.py
View File

@@ -35,7 +35,7 @@ from awx.main.models.mixins import (
CustomVirtualEnvMixin, CustomVirtualEnvMixin,
RelatedJobsMixin RelatedJobsMixin
) )
from awx.main.utils import update_scm_url from awx.main.utils import update_scm_url, polymorphic
from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook from awx.main.utils.ansible import skip_directory, could_be_inventory, could_be_playbook
from awx.main.fields import ImplicitRoleField from awx.main.fields import ImplicitRoleField
from awx.main.models.rbac import ( from awx.main.models.rbac import (
@@ -272,7 +272,7 @@ class Project(UnifiedJobTemplate, ProjectOptions, ResourceMixin, CustomVirtualEn
null=True, null=True,
blank=True, blank=True,
default=None, default=None,
on_delete=models.SET_NULL, on_delete=polymorphic.SET_NULL,
related_name='+', related_name='+',
help_text=_('The default execution environment for jobs run using this project.'), help_text=_('The default execution environment for jobs run using this project.'),
) )

5
awx/settings/defaults.py
View File

@@ -1063,6 +1063,11 @@ LOGGING = {
'level': 'INFO', 'level': 'INFO',
'propagate': False 'propagate': False
}, },
'awx.analytics.performance': {
'handlers': ['console', 'file', 'tower_warnings', 'external_logger'],
'level': 'DEBUG',
'propagate': False
},
'awx.analytics.job_lifecycle': { 'awx.analytics.job_lifecycle': {
'handlers': ['console', 'job_lifecycle'], 'handlers': ['console', 'job_lifecycle'],
'level': 'DEBUG', 'level': 'DEBUG',

14
awx_collection/plugins/modules/tower_user.py
View File

@@ -149,19 +149,19 @@ def main():
# Create the data that gets sent for create and update # Create the data that gets sent for create and update
new_fields = {} new_fields = {}
if username: if username is not None:
new_fields['username'] = module.get_item_name(existing_item) if existing_item else username new_fields['username'] = module.get_item_name(existing_item) if existing_item else username
if first_name: if first_name is not None:
new_fields['first_name'] = first_name new_fields['first_name'] = first_name
if last_name: if last_name is not None:
new_fields['last_name'] = last_name new_fields['last_name'] = last_name
if email: if email is not None:
new_fields['email'] = email new_fields['email'] = email
if is_superuser: if is_superuser is not None:
new_fields['is_superuser'] = is_superuser new_fields['is_superuser'] = is_superuser
if is_system_auditor: if is_system_auditor is not None:
new_fields['is_system_auditor'] = is_system_auditor new_fields['is_system_auditor'] = is_system_auditor
if password: if password is not None:
new_fields['password'] = password new_fields['password'] = password
# If the state was present and we can let the module build or update the existing item, this will return on its own # If the state was present and we can let the module build or update the existing item, this will return on its own

20
awx_collection/test/awx/test_user.py
View File

@@ -57,3 +57,23 @@ def test_update_password_on_create(run_module, admin_user, mock_auth_stuff):
assert not result.get('failed', False), result.get('msg', result) assert not result.get('failed', False), result.get('msg', result)
assert not result.get('changed') assert not result.get('changed')
@pytest.mark.django_db
def test_update_user(run_module, admin_user, mock_auth_stuff):
result = run_module('tower_user', dict(
username='Bob',
password='pass4word',
is_system_auditor=True
), admin_user)
assert not result.get('failed', False), result.get('msg', result)
assert result.get('changed'), result
update_result = run_module('tower_user', dict(
username='Bob',
is_system_auditor=False
), admin_user)
assert update_result.get('changed')
user = User.objects.get(id=result['id'])
assert not user.is_system_auditor

11
awx_collection/tests/integration/targets/tower_job_launch/tasks/main.yml
View File

@@ -164,10 +164,17 @@
that: that:
- "result is changed" - "result is changed"
- name: Wait for 2nd job max 120s - name: Wait for a job template to complete
tower_job_wait: tower_job_wait:
job_id: "{{result.id}}" job_id: "{{ result.id }}"
max_interval: 10
timeout: 120 timeout: 120
register: result
- assert:
that:
- "result is not changed"
- "result.status == 'successful'"
- name: Get the job - name: Get the job
tower_job_list: tower_job_list:

2
awxkit/setup.py
View File

@@ -70,7 +70,7 @@ setup(
python_requires=">=3.6", python_requires=">=3.6",
extras_require={ extras_require={
'formatting': ['jq'], 'formatting': ['jq'],
'websockets': ['websocket-client>0.54.0'], 'websockets': ['websocket-client==0.57.0'],
'crypto': ['cryptography'] 'crypto': ['cryptography']
}, },
license='Apache 2.0', license='Apache 2.0',

28
docs/licenses/click.txt Normal file
View File

@@ -0,0 +1,28 @@
Copyright 2014 Pallets
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
1. Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

22
requirements/README.md
View File

@@ -1,15 +1,11 @@
# Dependency Management # Dependency Management
The `requirements.txt` and `requirements_ansible.txt` files are generated from `requirements.in` and `requirements_ansible.in`, respectively, using `pip-tools` `pip-compile`. The `requirements.txt` file is generated from `requirements.in`, using `pip-tools` `pip-compile`.
## How To Use ## How To Use
Commands should be run from inside the `./requirements` directory of the awx repository. Commands should be run from inside the `./requirements` directory of the awx repository.
Make sure you have `patch, awk, python3, python2, python3-venv, python2-virtualenv, pip2, pip3` installed. The development container image should have all these.
Even in the dev container, you may still have to dnf install `libpq-devel libcurl-devel`.
### Upgrading or Adding Select Libraries ### Upgrading or Adding Select Libraries
If you need to add or upgrade one targeted library, then modify `requirements.in`, If you need to add or upgrade one targeted library, then modify `requirements.in`,
@@ -17,6 +13,9 @@ then run the script:
`./updater.sh` `./updater.sh`
NOTE: `./updater.sh` uses /usr/bin/python3.6, to match the current python version
(3.6) used to build releases.
#### Upgrading Unpinned Dependency #### Upgrading Unpinned Dependency
If you require a new version of a dependency that does not have a pinned version If you require a new version of a dependency that does not have a pinned version
@@ -33,14 +32,6 @@ You can upgrade (`pip-compile --upgrade`) the dependencies by running
`./updater.sh upgrade`. `./updater.sh upgrade`.
## What The Script Does
This script will:
- Update `requirements.txt` based on `requirements.in`
- Update/generate `requirements_ansible.txt` based on `requirements_ansible.in`
- including an automated patch that adds `python_version < "3"` for Python 2 backward compatibility
## Licenses and Source Files ## Licenses and Source Files
If any library has a change to its license with the upgrade, then the license for that library If any library has a change to its license with the upgrade, then the license for that library
@@ -129,11 +120,6 @@ This breaks a very large amount of AWX code that assumes these fields
are returned as dicts. Upgrading this library will require a refactor are returned as dicts. Upgrading this library will require a refactor
to accomidate this change. to accomidate this change.
### wheel
azure-cli-core requires a version of wheel that is incompatible with
certain packages building with later versions of pip, so we override it.
### pip and setuptools ### pip and setuptools
The offline installer needs to have functionality confirmed before upgrading these. The offline installer needs to have functionality confirmed before upgrading these.

561
requirements/requirements.txt
View File

@@ -1,135 +1,430 @@
adal==1.2.2 # via msrestazure adal==1.2.2
aiohttp==3.6.2 # via -r /awx_devel/requirements/requirements.in # via msrestazure
aioredis==1.3.1 # via channels-redis aiohttp==3.6.2
# ansible-runner==1.4.7 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
ansiconv==1.0.0 # via -r /awx_devel/requirements/requirements.in aioredis==1.3.1
asciichartpy==1.5.25 # via -r /awx_devel/requirements/requirements.in # via channels-redis
asgiref==3.2.5 # via channels, channels-redis, daphne #ansible-runner==1.4.7
async-timeout==3.0.1 # via aiohttp, aioredis # via
attrs==19.3.0 # via aiohttp, automat, jsonschema, service-identity, twisted # -r /awx_devel/requirements/requirements.in
autobahn==20.12.3 # via -r /awx_devel/requirements/requirements.in, daphne # -r /awx_devel/requirements/requirements_git.txt
automat==20.2.0 # via twisted ansiconv==1.0.0
azure-common==1.1.25 # via azure-keyvault # via -r /awx_devel/requirements/requirements.in
azure-keyvault==1.1.0 # via -r /awx_devel/requirements/requirements.in asciichartpy==1.5.25
azure-nspkg==3.0.2 # via azure-keyvault # via -r /awx_devel/requirements/requirements.in
cachetools==4.0.0 # via google-auth asgiref==3.2.5
# don't add certifi to our requirements, because we have our own specific implementation: https://github.com/ansible/system-certifi # via
cffi==1.14.0 # via cryptography # channels
channels-redis==3.1.0 # via -r /awx_devel/requirements/requirements.in # channels-redis
channels==2.4.0 # via -r /awx_devel/requirements/requirements.in, channels-redis # daphne
chardet==3.0.4 # via aiohttp, requests async-timeout==3.0.1
constantly==15.1.0 # via twisted # via
cryptography==2.9.2 # via -r /awx_devel/requirements/requirements.in, adal, autobahn, azure-keyvault, pyopenssl, service-identity, social-auth-core # aiohttp
daphne==2.4.1 # via -r /awx_devel/requirements/requirements.in, channels # aioredis
defusedxml==0.6.0 # via python3-openid, python3-saml, social-auth-core attrs==19.3.0
dictdiffer==0.8.1 # via openshift # via
django-auth-ldap==2.1.0 # via -r /awx_devel/requirements/requirements.in # aiohttp
django-cors-headers==3.2.1 # via -r /awx_devel/requirements/requirements.in # automat
django-crum==0.7.5 # via -r /awx_devel/requirements/requirements.in # jsonschema
django-extensions==2.2.9 # via -r /awx_devel/requirements/requirements.in # service-identity
django-guid==2.2.0 # via -r /awx_devel/requirements/requirements.in # twisted
django-jsonfield==1.2.0 # via -r /awx_devel/requirements/requirements.in autobahn==20.12.3
django-oauth-toolkit==1.1.3 # via -r /awx_devel/requirements/requirements.in # via
django-pglocks==1.0.4 # via -r /awx_devel/requirements/requirements.in # -r /awx_devel/requirements/requirements.in
django-polymorphic==2.1.2 # via -r /awx_devel/requirements/requirements.in # daphne
django-qsstats-magic==1.1.0 # via -r /awx_devel/requirements/requirements.in automat==20.2.0
django-radius==1.3.3 # via -r /awx_devel/requirements/requirements.in # via twisted
django-redis==4.5.0 # via -r /awx_devel/requirements/requirements.in azure-common==1.1.25
django-solo==1.1.3 # via -r /awx_devel/requirements/requirements.in # via azure-keyvault
django-split-settings==1.0.0 # via -r /awx_devel/requirements/requirements.in azure-keyvault==1.1.0
django-taggit==1.2.0 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
django==2.2.16 # via -r /awx_devel/requirements/requirements.in, channels, django-auth-ldap, django-cors-headers, django-crum, django-guid, django-jsonfield, django-oauth-toolkit, django-polymorphic, django-taggit, djangorestframework azure-nspkg==3.0.2
djangorestframework-yaml==1.0.3 # via -r /awx_devel/requirements/requirements.in # via azure-keyvault
djangorestframework==3.12.1 # via -r /awx_devel/requirements/requirements.in cachetools==4.0.0
docutils==0.16 # via python-daemon # via google-auth
future==0.16.0 # via django-radius # via
gitdb==4.0.2 # via gitpython # -r /awx_devel/requirements/requirements_git.txt
gitpython==3.1.7 # via -r /awx_devel/requirements/requirements.in # kubernetes
google-auth==1.11.3 # via kubernetes # msrest
hiredis==1.0.1 # via aioredis # requests
hyperlink==20.0.1 # via autobahn, twisted cffi==1.14.0
idna-ssl==1.1.0 # via aiohttp # via cryptography
idna==2.9 # via hyperlink, idna-ssl, requests, twisted, yarl channels-redis==3.1.0
importlib-metadata==1.5.0 # via importlib-resources, irc, jsonschema # via -r /awx_devel/requirements/requirements.in
importlib-resources==1.4.0 # via jaraco.text channels==2.4.0
incremental==17.5.0 # via twisted # via
irc==18.0.0 # via -r /awx_devel/requirements/requirements.in # -r /awx_devel/requirements/requirements.in
isodate==0.6.0 # via msrest, python3-saml # channels-redis
jaraco.classes==3.1.0 # via jaraco.collections chardet==3.0.4
jaraco.collections==3.0.0 # via irc # via
jaraco.functools==3.0.0 # via irc, jaraco.text, tempora # aiohttp
jaraco.logging==3.0.0 # via irc # requests
jaraco.stream==3.0.0 # via irc click==7.1.2
jaraco.text==3.2.0 # via irc, jaraco.collections # via receptorctl
jinja2==2.11.2 # via -r /awx_devel/requirements/requirements.in, openshift constantly==15.1.0
json-log-formatter==0.3.0 # via -r /awx_devel/requirements/requirements.in # via twisted
jsonschema==3.2.0 # via -r /awx_devel/requirements/requirements.in cryptography==2.9.2
kubernetes==11.0.0 # via openshift # via
lockfile==0.12.2 # via python-daemon # -r /awx_devel/requirements/requirements.in
lxml==4.5.0 # via xmlsec # adal
markdown==3.2.1 # via -r /awx_devel/requirements/requirements.in # autobahn
markupsafe==1.1.1 # via jinja2 # azure-keyvault
more-itertools==8.2.0 # via irc, jaraco.classes, jaraco.functools # pyopenssl
msgpack==1.0.0 # via channels-redis # service-identity
msrest==0.6.11 # via azure-keyvault, msrestazure # social-auth-core
msrestazure==0.6.3 # via azure-keyvault daphne==2.4.1
multidict==4.7.5 # via aiohttp, yarl # via
netaddr==0.7.19 # via pyrad # -r /awx_devel/requirements/requirements.in
oauthlib==3.1.0 # via django-oauth-toolkit, requests-oauthlib, social-auth-core # channels
openshift==0.11.0 # via -r /awx_devel/requirements/requirements.in defusedxml==0.6.0
pexpect==4.7.0 # via -r /awx_devel/requirements/requirements.in, ansible-runner # via
pkgconfig==1.5.1 # via xmlsec # python3-openid
prometheus-client==0.7.1 # via -r /awx_devel/requirements/requirements.in # python3-saml
psutil==5.7.0 # via ansible-runner # social-auth-core
psycopg2==2.8.4 # via -r /awx_devel/requirements/requirements.in dictdiffer==0.8.1
ptyprocess==0.6.0 # via pexpect # via openshift
pyasn1-modules==0.2.8 # via google-auth, python-ldap, service-identity django-auth-ldap==2.1.0
pyasn1==0.4.8 # via pyasn1-modules, python-ldap, rsa, service-identity # via -r /awx_devel/requirements/requirements.in
pycparser==2.20 # via cffi django-cors-headers==3.2.1
pygerduty==0.38.2 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
pyhamcrest==2.0.2 # via twisted django-crum==0.7.5
pyjwt==1.7.1 # via adal, social-auth-core, twilio # via -r /awx_devel/requirements/requirements.in
pyopenssl==19.1.0 # via twisted django-extensions==2.2.9
pyparsing==2.4.6 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
pyrad==2.3 # via django-radius django-guid==2.2.0
pyrsistent==0.15.7 # via jsonschema # via -r /awx_devel/requirements/requirements.in
python-daemon==2.2.4 # via ansible-runner django-jsonfield==1.2.0
python-dateutil==2.8.1 # via adal, kubernetes # via -r /awx_devel/requirements/requirements.in
python-ldap==3.3.1 # via -r /awx_devel/requirements/requirements.in, django-auth-ldap django-oauth-toolkit==1.1.3
python-string-utils==1.0.0 # via openshift # via -r /awx_devel/requirements/requirements.in
python3-openid==3.1.0 # via social-auth-core django-pglocks==1.0.4
python3-saml==1.9.0 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
pytz==2019.3 # via django, irc, tempora, twilio django-polymorphic==2.1.2
pyyaml==5.4.1 # via -r /awx_devel/requirements/requirements.in, ansible-runner, djangorestframework-yaml, kubernetes # via -r /awx_devel/requirements/requirements.in
redis==3.4.1 # via -r /awx_devel/requirements/requirements.in, django-redis django-qsstats-magic==1.1.0
requests-oauthlib==1.3.0 # via kubernetes, msrest, social-auth-core # via -r /awx_devel/requirements/requirements.in
requests==2.23.0 # via -r /awx_devel/requirements/requirements.in, adal, azure-keyvault, django-oauth-toolkit, kubernetes, msrest, requests-oauthlib, slackclient, social-auth-core, twilio django-radius==1.3.3
rsa==4.0 # via google-auth # via -r /awx_devel/requirements/requirements.in
ruamel.yaml.clib==0.2.0 # via ruamel.yaml django-redis==4.5.0
ruamel.yaml==0.16.10 # via openshift # via -r /awx_devel/requirements/requirements.in
schedule==0.6.0 # via -r /awx_devel/requirements/requirements.in django-solo==1.1.3
service-identity==18.1.0 # via twisted # via -r /awx_devel/requirements/requirements.in
six==1.14.0 # via ansible-runner, automat, cryptography, django-extensions, django-pglocks, google-auth, isodate, jaraco.collections, jaraco.logging, jaraco.text, jsonschema, kubernetes, openshift, pygerduty, pyopenssl, pyrad, pyrsistent, python-dateutil, slackclient, social-auth-app-django, social-auth-core, tacacs-plus, twilio, websocket-client django-split-settings==1.0.0
slackclient==1.1.2 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
smmap==3.0.1 # via gitdb django-taggit==1.2.0
social-auth-app-django==3.1.0 # via -r /awx_devel/requirements/requirements.in # via -r /awx_devel/requirements/requirements.in
social-auth-core==3.3.1 # via -r /awx_devel/requirements/requirements.in, social-auth-app-django django==2.2.16
sqlparse==0.3.1 # via django # via
tacacs_plus==1.0 # via -r /awx_devel/requirements/requirements.in # -r /awx_devel/requirements/requirements.in
tempora==2.1.0 # via irc, jaraco.logging # channels
twilio==6.37.0 # via -r /awx_devel/requirements/requirements.in # django-auth-ldap
twisted[tls]==20.3.0 # via -r /awx_devel/requirements/requirements.in, daphne # django-cors-headers
txaio==20.12.1 # via autobahn # django-crum
typing-extensions==3.7.4.1 # via aiohttp # django-guid
urllib3==1.25.8 # via kubernetes, requests # django-jsonfield
uwsgi==2.0.18 # via -r /awx_devel/requirements/requirements.in # django-oauth-toolkit
uwsgitop==0.11 # via -r /awx_devel/requirements/requirements.in # django-polymorphic
websocket-client==0.57.0 # via kubernetes, slackclient # django-taggit
xmlsec==1.3.3 # via python3-saml # djangorestframework
yarl==1.4.2 # via aiohttp djangorestframework-yaml==1.0.3
zipp==3.1.0 # via importlib-metadata, importlib-resources # via -r /awx_devel/requirements/requirements.in
zope.interface==5.0.0 # via twisted djangorestframework==3.12.1
# via -r /awx_devel/requirements/requirements.in
docutils==0.16
# via python-daemon
future==0.16.0
# via django-radius
gitdb==4.0.2
# via gitpython
gitpython==3.1.7
# via -r /awx_devel/requirements/requirements.in
google-auth==1.11.3
# via kubernetes
hiredis==1.0.1
# via aioredis
hyperlink==20.0.1
# via
# autobahn
# twisted
idna-ssl==1.1.0
# via aiohttp
idna==2.9
# via
# hyperlink
# idna-ssl
# requests
# twisted
# yarl
importlib-metadata==1.5.0
# via
# importlib-resources
# irc
# jsonschema
importlib-resources==1.4.0
# via jaraco.text
incremental==17.5.0
# via twisted
irc==18.0.0
# via -r /awx_devel/requirements/requirements.in
isodate==0.6.0
# via
# msrest
# python3-saml
jaraco.classes==3.1.0
# via jaraco.collections
jaraco.collections==3.0.0
# via irc
jaraco.functools==3.0.0
# via
# irc
# jaraco.text
# tempora
jaraco.logging==3.0.0
# via irc
jaraco.stream==3.0.0
# via irc
jaraco.text==3.2.0
# via
# irc
# jaraco.collections
jinja2==2.11.2
# via
# -r /awx_devel/requirements/requirements.in
# openshift
json-log-formatter==0.3.0
# via -r /awx_devel/requirements/requirements.in
jsonschema==3.2.0
# via -r /awx_devel/requirements/requirements.in
kubernetes==11.0.0
# via openshift
lockfile==0.12.2
# via python-daemon
lxml==4.5.0
# via xmlsec
markdown==3.2.1
# via -r /awx_devel/requirements/requirements.in
markupsafe==1.1.1
# via jinja2
more-itertools==8.2.0
# via
# irc
# jaraco.classes
# jaraco.functools
msgpack==1.0.0
# via channels-redis
msrest==0.6.11
# via
# azure-keyvault
# msrestazure
msrestazure==0.6.3
# via azure-keyvault
multidict==4.7.5
# via
# aiohttp
# yarl
netaddr==0.7.19
# via pyrad
oauthlib==3.1.0
# via
# django-oauth-toolkit
# requests-oauthlib
# social-auth-core
openshift==0.11.0
# via -r /awx_devel/requirements/requirements.in
pexpect==4.7.0
# via
# -r /awx_devel/requirements/requirements.in
# ansible-runner
pkgconfig==1.5.1
# via xmlsec
prometheus-client==0.7.1
# via -r /awx_devel/requirements/requirements.in
psutil==5.7.0
# via ansible-runner
psycopg2==2.8.4
# via -r /awx_devel/requirements/requirements.in
ptyprocess==0.6.0
# via pexpect
pyasn1-modules==0.2.8
# via
# google-auth
# python-ldap
# service-identity
pyasn1==0.4.8
# via
# pyasn1-modules
# python-ldap
# rsa
# service-identity
pycparser==2.20
# via cffi
pygerduty==0.38.2
# via -r /awx_devel/requirements/requirements.in
pyhamcrest==2.0.2
# via twisted
pyjwt==1.7.1
# via
# adal
# social-auth-core
# twilio
pyopenssl==19.1.0
# via twisted
pyparsing==2.4.6
# via -r /awx_devel/requirements/requirements.in
pyrad==2.3
# via django-radius
pyrsistent==0.15.7
# via jsonschema
python-daemon==2.2.4
# via ansible-runner
python-dateutil==2.8.1
# via
# adal
# kubernetes
# receptorctl
python-ldap==3.3.1
# via
# -r /awx_devel/requirements/requirements.in
# django-auth-ldap
python-string-utils==1.0.0
# via openshift
python3-openid==3.1.0
# via social-auth-core
python3-saml==1.9.0
# via -r /awx_devel/requirements/requirements.in
pytz==2019.3
# via
# django
# irc
# tempora
# twilio
pyyaml==5.4.1
# via
# -r /awx_devel/requirements/requirements.in
# ansible-runner
# djangorestframework-yaml
# kubernetes
# receptorctl
# via -r /awx_devel/requirements/requirements_git.txt
redis==3.4.1
# via
# -r /awx_devel/requirements/requirements.in
# django-redis
requests-oauthlib==1.3.0
# via
# kubernetes
# msrest
# social-auth-core
requests==2.23.0
# via
# -r /awx_devel/requirements/requirements.in
# adal
# azure-keyvault
# django-oauth-toolkit
# kubernetes
# msrest
# requests-oauthlib
# slackclient
# social-auth-core
# twilio
rsa==4.0
# via google-auth
ruamel.yaml.clib==0.2.0
# via ruamel.yaml
ruamel.yaml==0.16.10
# via openshift
schedule==0.6.0
# via -r /awx_devel/requirements/requirements.in
service-identity==18.1.0
# via twisted
six==1.14.0
# via
# ansible-runner
# automat
# cryptography
# django-extensions
# django-pglocks
# google-auth
# isodate
# jaraco.collections
# jaraco.logging
# jaraco.text
# jsonschema
# kubernetes
# openshift
# pygerduty
# pyopenssl
# pyrad
# pyrsistent
# python-dateutil
# slackclient
# social-auth-app-django
# social-auth-core
# tacacs-plus
# twilio
# websocket-client
slackclient==1.1.2
# via -r /awx_devel/requirements/requirements.in
smmap==3.0.1
# via gitdb
social-auth-app-django==3.1.0
# via -r /awx_devel/requirements/requirements.in
social-auth-core==3.3.1
# via
# -r /awx_devel/requirements/requirements.in
# social-auth-app-django
sqlparse==0.3.1
# via django
tacacs_plus==1.0
# via -r /awx_devel/requirements/requirements.in
tempora==2.1.0
# via
# irc
# jaraco.logging
twilio==6.37.0
# via -r /awx_devel/requirements/requirements.in
twisted[tls]==20.3.0
# via
# -r /awx_devel/requirements/requirements.in
# daphne
txaio==20.12.1
# via autobahn
typing-extensions==3.7.4.1
# via aiohttp
urllib3==1.25.8
# via
# kubernetes
# requests
uwsgi==2.0.18
# via -r /awx_devel/requirements/requirements.in
uwsgitop==0.11
# via -r /awx_devel/requirements/requirements.in
websocket-client==0.57.0
# via
# kubernetes
# slackclient
xmlsec==1.3.3
# via python3-saml
yarl==1.4.2
# via aiohttp
zipp==3.1.0
# via
# importlib-metadata
# importlib-resources
zope.interface==5.0.0
# via twisted
# The following packages are considered to be unsafe in a requirements file: # The following packages are considered to be unsafe in a requirements file:
pip==19.3.1 # via -r /awx_devel/requirements/requirements.in pip==19.3.1
setuptools==41.6.0 # via -r /awx_devel/requirements/requirements.in, asciichartpy, google-auth, jsonschema, kubernetes, markdown, python-daemon, zope.interface # via -r /awx_devel/requirements/requirements.in
setuptools==41.6.0
# via
# -r /awx_devel/requirements/requirements.in
# asciichartpy
# google-auth
# jsonschema
# kubernetes
# markdown
# python-daemon
# receptorctl
# zope.interface

2
requirements/requirements_git.txt
View File

@@ -1,3 +1,3 @@
git+https://github.com/ansible/system-certifi.git@devel#egg=certifi git+https://github.com/ansible/system-certifi.git@devel#egg=certifi
git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner git+git://github.com/ansible/ansible-runner@devel#egg=ansible-runner
git+https://github.com/project-receptor/receptor.git@#egg=receptorctl&subdirectory=receptorctl git+https://github.com/project-receptor/receptor.git@0.9.6#egg=receptorctl&subdirectory=receptorctl

63
requirements/updater.sh
View File

@@ -2,71 +2,32 @@
set -ue set -ue
requirements_in="$(readlink -f ./requirements.in)" requirements_in="$(readlink -f ./requirements.in)"
requirements_ansible_in="$(readlink -f ./requirements_ansible.in)"
requirements="$(readlink -f ./requirements.txt)" requirements="$(readlink -f ./requirements.txt)"
requirements_git="$(readlink -f ./requirements_git.txt)" requirements_git="$(readlink -f ./requirements_git.txt)"
requirements_ansible="$(readlink -f ./requirements_ansible.txt)"
pip_compile="pip-compile --no-header --quiet -r --allow-unsafe" pip_compile="pip-compile --no-header --quiet -r --allow-unsafe"
check_prerequisites() {
for thing in patch awk python3 python2 virtualenv ; do
command -v $thing >/dev/null 2>&1 || { echo "$thing not installed or available. Please fix this before running." ; exit 1 ; }
done
}
_cleanup() { _cleanup() {
cd / cd /
test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}" test "${KEEP_TMP:-0}" = 1 || rm -rf "${_tmp}"
} }
install_deps() { generate_requirements() {
pip install pip --upgrade venv="`pwd`/venv"
pip install "pip-tools==5.4.0" # see https://github.com/jazzband/pip-tools/pull/1237 echo $venv
} /usr/bin/python3.6 -m venv "${venv}"
generate_requirements_v3() {
venv="./venv3"
python3 -m venv "${venv}"
# shellcheck disable=SC1090 # shellcheck disable=SC1090
. "${venv}/bin/activate" source ${venv}/bin/activate
install_deps ${venv}/bin/python3.6 -m pip install -U pip pip-tools
${pip_compile} --output-file requirements.txt "${requirements_in}" "${requirements_git}" ${pip_compile} --output-file requirements.txt "${requirements_in}" "${requirements_git}"
# consider the git requirements for purposes of resolving deps # consider the git requirements for purposes of resolving deps
# Then remove any git+ lines from requirements.txt # Then remove any git+ lines from requirements.txt
cp requirements.txt requirements_tmp.txt cp requirements.txt requirements_tmp.txt
grep -v "^git+" requirements_tmp.txt > requirements.txt && rm requirements_tmp.txt grep -v "^git+" requirements_tmp.txt > requirements.txt && rm requirements_tmp.txt
${pip_compile} --output-file requirements_ansible_py3.txt "${requirements_ansible_in}"
}
generate_requirements_v2() {
venv="./venv2"
virtualenv -p python2 "${venv}"
# shellcheck disable=SC1090
PS1="" . "${venv}/bin/activate"
install_deps
${pip_compile} --output-file requirements_ansible.txt "${requirements_ansible_in}"
}
generate_patch() {
a="requirements_ansible_py3.txt"
b="requirements_ansible.txt"
replace='; python_version < "3" #'
# most elegant/quick solution I could come up for now
out="$(diff --ignore-matching-lines='^#' --unified "${a}" "${b}" | \
awk -v replace="${replace}" '{ if (/^+\w/){ $2=replace; print;} else print; }' | \
sed 's/ ;/;/g')"
test -n "${out}"
echo "${out}"
} }
main() { main() {
check_prerequisites
_tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)" _tmp="$(mktemp -d --suffix .awx-requirements XXXX -p /tmp)"
trap _cleanup INT TERM EXIT trap _cleanup INT TERM EXIT
@@ -74,19 +35,11 @@ main() {
pip_compile="${pip_compile} --upgrade" pip_compile="${pip_compile} --upgrade"
fi fi
cp -vf requirements.txt requirements_ansible.txt "${_tmp}" cp -vf requirements.txt "${_tmp}"
cp -vf requirements_ansible.txt "${_tmp}/requirements_ansible_py3.txt"
cd "${_tmp}" cd "${_tmp}"
generate_requirements_v3 generate_requirements
generate_requirements_v2
sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible.txt
sed -i 's/^wheel==0.30.0.*/wheel==0.33.6 # via azure-cli-core (overriden, see upgrade blockers)/g' requirements_ansible_py3.txt
generate_patch | patch -p4 requirements_ansible_py3.txt
cp -vf requirements_ansible_py3.txt "${requirements_ansible}"
cp -vf requirements.txt "${requirements}" cp -vf requirements.txt "${requirements}"
_cleanup _cleanup

1
tools/ansible/roles/dockerfile/templates/Dockerfile.j2
View File

@@ -141,6 +141,7 @@ RUN dnf -y install \
strace \ strace \
vim \ vim \
nmap-ncat \ nmap-ncat \
libpq-devel \
nodejs \ nodejs \
nss \ nss \
make \ make \