Compare commits

...

7 Commits

Author SHA1 Message Date
Alan Rominger
918d5b3565 Do some aesthetic adjustments to role presentation fields (#15153)
* Do some asthetic adjustments to role presentation fields

* Correctly test managed setup

* Minor migration adjustments
2024-04-29 17:11:10 -04:00
Hao Liu
158314af50 Delete deprecated Cypress UI e2e_test.yml (#15155)
Delete e2e_test.yml

Remove because it's no longer being maintained
2024-04-29 12:58:10 -04:00
Seth Foster
4754819a09 awx modules wait on event processing finished (#15152)
This change makes "wait: true" for jobs and syncs
look at the event_processing_finished instead of
finished field.

Right now there is a race condition where
a module might try to delete an inventory, but the events
for an inventory sync have not yet finished. We have a
RelatedJobsPreventDeleteMixin that checks for this condition.

bulk jobs don't have event_processing_finished so we just
use finished field in that case.
2024-04-26 17:33:34 -04:00
Seth Foster
78fc23138a Pin openssl 3.0.7 (#15147)
followup to PR #15142

This commit pins openssl in the awx image,
not just the builder image.
2024-04-26 12:29:22 -04:00
Alan Rominger
014534bfa5 Upgrade DRF (#15144)
* Upgrade DRF

* Fix failures caused by DRF upgrade
2024-04-25 15:37:08 -04:00
Seth Foster
2502e7c7d8 Temporarily downgrade openssl (#15142)
openssl 3.2.0 has incompatiblity issues with
the libpq version we are using, and causes
some C runtime errors:
"double free or corruption (out)"

see awx issue #15136

also this issue

github.com/conan-io/conan-center-index/pull/22615

once the libpq libraries on centos stream9 are
updated with the patch, we can unpin openssl

Signed-off-by: Seth Foster <fosterbseth@gmail.com>
2024-04-25 14:01:03 -04:00
Jeff Bradberry
fb237e3834 Stop pre-caching every resource in the system upon import
If we don't have something in the cache when we call
get_by_natural_key, do an actual filtered query for it and cache the
results.  We'll get more overall API calls this way, but they'll be
smaller and will happen while we are importing, not upfront.
2024-04-24 17:04:40 -04:00
18 changed files with 135 additions and 178 deletions

View File

@@ -1,75 +0,0 @@
---
name: E2E Tests
env:
LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
on:
pull_request_target:
types: [labeled]
jobs:
e2e-test:
if: contains(github.event.pull_request.labels.*.name, 'qe:e2e')
runs-on: ubuntu-latest
timeout-minutes: 40
permissions:
packages: write
contents: read
strategy:
matrix:
job: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/run_awx_devel
id: awx
with:
build-ui: true
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Pull awx_cypress_base image
run: |
docker pull quay.io/awx/awx_cypress_base:latest
- name: Checkout test project
uses: actions/checkout@v3
with:
repository: ${{ github.repository_owner }}/tower-qa
ssh-key: ${{ secrets.QA_REPO_KEY }}
path: tower-qa
ref: devel
- name: Build cypress
run: |
cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
docker build -t awx-pf-tests .
- name: Run E2E tests
env:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
run: |
export COMMIT_INFO_BRANCH=$GITHUB_HEAD_REF
export COMMIT_INFO_AUTHOR=$GITHUB_ACTOR
export COMMIT_INFO_SHA=$GITHUB_SHA
export COMMIT_INFO_REMOTE=$GITHUB_REPOSITORY_OWNER
cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
AWX_IP=${{ steps.awx.outputs.ip }}
printenv > .env
echo "Executing tests:"
docker run \
--network '_sources_default' \
--ipc=host \
--env-file=.env \
-e CYPRESS_baseUrl="https://$AWX_IP:8043" \
-e CYPRESS_AWX_E2E_USERNAME=admin \
-e CYPRESS_AWX_E2E_PASSWORD='password' \
-e COMMAND="npm run cypress-concurrently-gha" \
-v /dev/shm:/dev/shm \
-v $PWD:/e2e \
-w /e2e \
awx-pf-tests run --project .
- uses: ./.github/actions/upload_awx_devel_logs
if: always()
with:
log-filename: e2e-${{ matrix.job }}.log

View File

@@ -61,6 +61,10 @@ class StringListBooleanField(ListField):
def to_representation(self, value):
try:
if isinstance(value, str):
# https://github.com/encode/django-rest-framework/commit/a180bde0fd965915718b070932418cabc831cee1
# DRF changed truthy and falsy lists to be capitalized
value = value.lower()
if isinstance(value, (list, tuple)):
return super(StringListBooleanField, self).to_representation(value)
elif value in BooleanField.TRUE_VALUES:
@@ -78,6 +82,8 @@ class StringListBooleanField(ListField):
def to_internal_value(self, data):
try:
if isinstance(data, str):
data = data.lower()
if isinstance(data, (list, tuple)):
return super(StringListBooleanField, self).to_internal_value(data)
elif data in BooleanField.TRUE_VALUES:

View File

@@ -275,7 +275,12 @@ def setup_managed_role_definitions(apps, schema_editor):
"""
Idepotent method to create or sync the managed role definitions
"""
to_create = settings.ANSIBLE_BASE_ROLE_PRECREATE
to_create = {
'object_admin': '{cls.__name__} Admin',
'org_admin': 'Organization Admin',
'org_children': 'Organization {cls.__name__} Admin',
'special': '{cls.__name__} {action}',
}
ContentType = apps.get_model('contenttypes', 'ContentType')
Permission = apps.get_model('dab_rbac', 'DABPermission')

View File

@@ -10,6 +10,9 @@ import re
# django-rest-framework
from rest_framework.serializers import ValidationError
# crum to impersonate users
from crum import impersonate
# Django
from django.db import models, transaction, connection
from django.db.models.signals import m2m_changed
@@ -553,17 +556,22 @@ def get_role_definition(role):
return
f = obj._meta.get_field(role.role_field)
action_name = f.name.rsplit("_", 1)[0]
rd_name = f'{type(obj).__name__} {action_name.title()} Compat'
model_print = type(obj).__name__
rd_name = f'{model_print} {action_name.title()} Compat'
perm_list = get_role_codenames(role)
defaults = {'content_type_id': role.content_type_id}
try:
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
except ValidationError:
# This is a tricky case - practically speaking, users should not be allowed to create team roles
# or roles that include the team member permission.
# If we need to create this for compatibility purposes then we will create it as a managed non-editable role
defaults['managed'] = True
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
defaults = {
'content_type_id': role.content_type_id,
'description': f'Has {action_name.title()} permission to {model_print} for backwards API compatibility',
}
with impersonate(None):
try:
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
except ValidationError:
# This is a tricky case - practically speaking, users should not be allowed to create team roles
# or roles that include the team member permission.
# If we need to create this for compatibility purposes then we will create it as a managed non-editable role
defaults['managed'] = True
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
return rd

View File

@@ -30,7 +30,7 @@ def test_idempotent_credential_type_setup():
@pytest.mark.django_db
def test_create_user_credential_via_credentials_list(post, get, alice, credentialtype_ssh):
def test_create_user_credential_via_credentials_list(post, get, alice, credentialtype_ssh, setup_managed_roles):
params = {
'credential_type': 1,
'inputs': {'username': 'someusername'},
@@ -81,7 +81,7 @@ def test_credential_validation_error_with_multiple_owner_fields(post, admin, ali
@pytest.mark.django_db
def test_create_user_credential_via_user_credentials_list(post, get, alice, credentialtype_ssh):
def test_create_user_credential_via_user_credentials_list(post, get, alice, credentialtype_ssh, setup_managed_roles):
params = {
'credential_type': 1,
'inputs': {'username': 'someusername'},

View File

@@ -16,6 +16,8 @@ from django.db.backends.sqlite3.base import SQLiteCursorWrapper
from django.db.models.signals import post_migrate
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
# AWX
from awx.main.models.projects import Project
from awx.main.models.ha import Instance
@@ -90,6 +92,12 @@ def deploy_jobtemplate(project, inventory, credential):
return jt
@pytest.fixture
def setup_managed_roles():
"Run the migration script to pre-create managed role definitions"
setup_managed_role_definitions(apps, None)
@pytest.fixture
def team(organization):
return organization.teams.create(name='test-team')

View File

@@ -1,10 +0,0 @@
import pytest
from django.apps import apps
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
@pytest.fixture
def managed_roles():
"Run the migration script to pre-create managed role definitions"
setup_managed_role_definitions(apps, None)

View File

@@ -1,45 +0,0 @@
import pytest
from django.apps import apps
from django.test.utils import override_settings
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
from ansible_base.rbac.models import RoleDefinition
INVENTORY_OBJ_PERMISSIONS = ['view_inventory', 'adhoc_inventory', 'use_inventory', 'change_inventory', 'delete_inventory', 'update_inventory']
@pytest.mark.django_db
def test_managed_definitions_precreate():
with override_settings(
ANSIBLE_BASE_ROLE_PRECREATE={
'object_admin': '{cls._meta.model_name}-admin',
'org_admin': 'organization-admin',
'org_children': 'organization-{cls._meta.model_name}-admin',
'special': '{cls._meta.model_name}-{action}',
}
):
setup_managed_role_definitions(apps, None)
rd = RoleDefinition.objects.get(name='inventory-admin')
assert rd.managed is True
# add permissions do not go in the object-level admin
assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)
# test org-level object admin permissions
rd = RoleDefinition.objects.get(name='organization-inventory-admin')
assert rd.managed is True
assert set(rd.permissions.values_list('codename', flat=True)) == set(['add_inventory', 'view_organization'] + INVENTORY_OBJ_PERMISSIONS)
@pytest.mark.django_db
def test_managed_definitions_custom_obj_admin_name():
with override_settings(
ANSIBLE_BASE_ROLE_PRECREATE={
'object_admin': 'foo-{cls._meta.model_name}-foo',
}
):
setup_managed_role_definitions(apps, None)
rd = RoleDefinition.objects.get(name='foo-inventory-foo')
assert rd.managed is True
# add permissions do not go in the object-level admin
assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)

View File

@@ -10,7 +10,7 @@ from ansible_base.rbac.models import RoleDefinition
@pytest.mark.django_db
def test_managed_roles_created(managed_roles):
def test_managed_roles_created(setup_managed_roles):
"Managed RoleDefinitions are created in post_migration signal, we expect to see them here"
for cls in (JobTemplate, Inventory):
ct = ContentType.objects.get_for_model(cls)
@@ -22,7 +22,7 @@ def test_managed_roles_created(managed_roles):
@pytest.mark.django_db
def test_custom_read_role(admin_user, post, managed_roles):
def test_custom_read_role(admin_user, post, setup_managed_roles):
rd_url = django_reverse('roledefinition-list')
resp = post(
url=rd_url, data={"name": "read role made for test", "content_type": "awx.inventory", "permissions": ['view_inventory']}, user=admin_user, expect=201
@@ -40,7 +40,7 @@ def test_custom_system_roles_prohibited(admin_user, post):
@pytest.mark.django_db
def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post, managed_roles):
def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post, setup_managed_roles):
"Alice can not see rando, and so can not give them a role assignment"
rd = RoleDefinition.objects.get(name='Inventory Admin')
rd.give_permission(alice, inventory)
@@ -51,7 +51,7 @@ def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post,
@pytest.mark.django_db
def test_assign_managed_role(admin_user, alice, rando, inventory, post, managed_roles, organization):
def test_assign_managed_role(admin_user, alice, rando, inventory, post, setup_managed_roles, organization):
rd = RoleDefinition.objects.get(name='Inventory Admin')
rd.give_permission(alice, inventory)
# When alice and rando are members of the same org, they can see each other
@@ -78,7 +78,7 @@ def test_assign_custom_delete_role(admin_user, rando, inventory, delete, patch):
@pytest.mark.django_db
def test_assign_custom_add_role(admin_user, rando, organization, post, managed_roles):
def test_assign_custom_add_role(admin_user, rando, organization, post, setup_managed_roles):
rd, _ = RoleDefinition.objects.get_or_create(
name='inventory-add', permissions=['add_inventory', 'view_organization'], content_type=ContentType.objects.get_for_model(Organization)
)

View File

@@ -2,11 +2,15 @@ from unittest import mock
import pytest
from django.contrib.contenttypes.models import ContentType
from crum import impersonate
from awx.main.models.rbac import get_role_from_object_role, give_creator_permissions
from awx.main.models import User, Organization, WorkflowJobTemplate, WorkflowJobTemplateNode, Team
from awx.api.versioning import reverse
from ansible_base.rbac.models import RoleUserAssignment
from ansible_base.rbac.models import RoleUserAssignment, RoleDefinition
@pytest.mark.django_db
@@ -14,7 +18,7 @@ from ansible_base.rbac.models import RoleUserAssignment
'role_name',
['execution_environment_admin_role', 'project_admin_role', 'admin_role', 'auditor_role', 'read_role', 'execute_role', 'notification_admin_role'],
)
def test_round_trip_roles(organization, rando, role_name, managed_roles):
def test_round_trip_roles(organization, rando, role_name, setup_managed_roles):
"""
Make an assignment with the old-style role,
get the equivelent new role
@@ -28,7 +32,39 @@ def test_round_trip_roles(organization, rando, role_name, managed_roles):
@pytest.mark.django_db
def test_organization_level_permissions(organization, inventory, managed_roles):
def test_role_naming(setup_managed_roles):
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='dmin')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Admin'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_action_role_naming(setup_managed_roles):
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='ecute')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Execute'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_compat_role_naming(setup_managed_roles, job_template, rando, alice):
with impersonate(alice):
job_template.read_role.members.add(rando)
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='ompat')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Read Compat'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_organization_level_permissions(organization, inventory, setup_managed_roles):
u1 = User.objects.create(username='alice')
u2 = User.objects.create(username='bob')
@@ -58,14 +94,14 @@ def test_organization_level_permissions(organization, inventory, managed_roles):
@pytest.mark.django_db
def test_organization_execute_role(organization, rando, managed_roles):
def test_organization_execute_role(organization, rando, setup_managed_roles):
organization.execute_role.members.add(rando)
assert rando in organization.execute_role
assert set(Organization.accessible_objects(rando, 'execute_role')) == set([organization])
@pytest.mark.django_db
def test_workflow_approval_list(get, post, admin_user, managed_roles):
def test_workflow_approval_list(get, post, admin_user, setup_managed_roles):
workflow_job_template = WorkflowJobTemplate.objects.create()
approval_node = WorkflowJobTemplateNode.objects.create(workflow_job_template=workflow_job_template)
url = reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': approval_node.pk, 'version': 'v2'})
@@ -79,14 +115,14 @@ def test_workflow_approval_list(get, post, admin_user, managed_roles):
@pytest.mark.django_db
def test_creator_permission(rando, admin_user, inventory, managed_roles):
def test_creator_permission(rando, admin_user, inventory, setup_managed_roles):
give_creator_permissions(rando, inventory)
assert rando in inventory.admin_role
assert rando in inventory.admin_role.members.all()
@pytest.mark.django_db
def test_team_team_read_role(rando, team, admin_user, post, managed_roles):
def test_team_team_read_role(rando, team, admin_user, post, setup_managed_roles):
orgs = [Organization.objects.create(name=f'foo-{i}') for i in range(2)]
teams = [Team.objects.create(name=f'foo-{i}', organization=orgs[i]) for i in range(2)]
teams[1].member_role.members.add(rando)

View File

@@ -165,7 +165,7 @@ class TestOrphanJobTemplate:
@pytest.mark.django_db
@pytest.mark.job_permissions
def test_job_template_creator_access(project, organization, rando, post):
def test_job_template_creator_access(project, organization, rando, post, setup_managed_roles):
project.use_role.members.add(rando)
response = post(
url=reverse('api:job_template_list'),

View File

@@ -1148,13 +1148,8 @@ ANSIBLE_BASE_CUSTOM_VIEW_PARENT = 'awx.api.generics.APIView'
# Settings for the ansible_base RBAC system
# Only used internally, names of the managed RoleDefinitions to create
ANSIBLE_BASE_ROLE_PRECREATE = {
'object_admin': '{cls.__name__} Admin',
'org_admin': 'Organization Admin',
'org_children': 'Organization {cls.__name__} Admin',
'special': '{cls.__name__} {action}',
}
# This has been moved to data migration code
ANSIBLE_BASE_ROLE_PRECREATE = {}
# Name for auto-created roles that give users permissions to what they create
ANSIBLE_BASE_ROLE_CREATOR_NAME = '{cls.__name__} Creator'

View File

@@ -1038,7 +1038,10 @@ class ControllerAPIModule(ControllerModule):
# Grab our start time to compare against for the timeout
start = time.time()
result = self.get_endpoint(url)
while not result['json']['finished']:
wait_on_field = 'event_processing_finished'
if wait_on_field not in result['json']:
wait_on_field = 'finished'
while not result['json'][wait_on_field]:
# If we are past our time out fail with a message
if timeout and timeout < time.time() - start:
# Account for Legacy messages

View File

@@ -234,7 +234,7 @@ class ApiV2(base.Base):
return endpoint.get(**{identifier: value}, all_pages=True)
def export_assets(self, **kwargs):
self._cache = page.PageCache()
self._cache = page.PageCache(self.connection)
# If no resource kwargs are explicitly used, export everything.
all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES)
@@ -335,7 +335,7 @@ class ApiV2(base.Base):
if name == 'roles':
indexed_roles = defaultdict(list)
for role in S:
if 'content_object' not in role:
if role.get('content_object') is None:
continue
indexed_roles[role['content_object']['type']].append(role)
self._roles.append((_page, indexed_roles))
@@ -411,7 +411,7 @@ class ApiV2(base.Base):
# FIXME: deal with pruning existing relations that do not match the import set
def import_assets(self, data):
self._cache = page.PageCache()
self._cache = page.PageCache(self.connection)
self._related = []
self._roles = []
@@ -420,11 +420,8 @@ class ApiV2(base.Base):
for resource in self._dependent_resources():
endpoint = getattr(self, resource)
# Load up existing objects, so that we can try to update or link to them
self._cache.get_page(endpoint)
imported = self._import_list(endpoint, data.get(resource) or [])
changed = changed or imported
# FIXME: should we delete existing unpatched assets?
self._assign_related()
self._assign_membership()

View File

@@ -11,6 +11,7 @@ from awxkit.utils import PseudoNamespace, is_relative_endpoint, are_same_endpoin
from awxkit.api import utils
from awxkit.api.client import Connection
from awxkit.api.registry import URLRegistry
from awxkit.api.resources import resources
from awxkit.config import config
import awxkit.exceptions as exc
@@ -493,10 +494,11 @@ class TentativePage(str):
class PageCache(object):
def __init__(self):
def __init__(self, connection=None):
self.options = {}
self.pages_by_url = {}
self.pages_by_natural_key = {}
self.connection = connection or Connection(config.base_url, not config.assume_untrusted)
def get_options(self, page):
url = page.endpoint if isinstance(page, Page) else str(page)
@@ -550,7 +552,31 @@ class PageCache(object):
return self.set_page(page)
def get_by_natural_key(self, natural_key):
endpoint = self.pages_by_natural_key.get(utils.freeze(natural_key))
log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), endpoint)
if endpoint:
return self.get_page(endpoint)
page = self.pages_by_natural_key.get(utils.freeze(natural_key))
if page is None:
# We need some way to get ahold of the top-level resource
# list endpoint from the natural_key type. The resources
# object more or less has that for each of the detail
# views. Just chop off the /<id>/ bit.
endpoint = getattr(resources, natural_key['type'], None)
if endpoint is None:
return
endpoint = ''.join([endpoint.rsplit('/', 2)[0], '/'])
page_type = get_registered_page(endpoint)
kwargs = {}
for k, v in natural_key.items():
if isinstance(v, str) and k != 'type':
kwargs[k] = v
# Do a filtered query against the list endpoint, usually
# with the name of the object but sometimes more.
list_page = page_type(self.connection, endpoint=endpoint).get(all_pages=True, **kwargs)
if 'results' in list_page:
for p in list_page.results:
self.set_page(p)
page = self.pages_by_natural_key.get(utils.freeze(natural_key))
log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), page)
if page:
return self.get_page(page)

View File

@@ -25,7 +25,7 @@ django-pglocks
django-radius
django-solo
django-split-settings==1.0.0 # We hit a strange issue where the release process errored when upgrading past 1.0.0 see UPGRADE BLOCKERS
djangorestframework
djangorestframework>=3.15.0
djangorestframework-yaml
filelock
GitPython>=3.1.37 # CVE-2023-41040

View File

@@ -167,7 +167,7 @@ django-split-settings==1.0.0
# via
# -r /awx_devel/requirements/requirements.in
# django-ansible-base
djangorestframework==3.14.0
djangorestframework==3.15.1
# via
# -r /awx_devel/requirements/requirements.in
# django-ansible-base
@@ -387,7 +387,6 @@ python3-openid==3.2.0
# via -r /awx_devel/requirements/requirements_git.txt
pytz==2024.1
# via
# djangorestframework
# irc
# tempora
pyyaml==6.0.1

View File

@@ -36,6 +36,8 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
{% endif %}
nss \
openldap-devel \
# pin to older openssl, see jira AAP-23449
openssl-3.0.7 \
patch \
postgresql \
postgresql-devel \
@@ -120,6 +122,8 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
krb5-workstation \
nginx \
"openldap >= 2.6.2-3" \
# pin to older openssl, see jira AAP-23449
openssl-3.0.7 \
postgresql \
python3.11 \
"python3.11-devel" \