Compare commits

..

3 Commits

Author SHA1 Message Date
Elijah DeLee
19e3cba35c Merge branch 'devel' into x-request-id 2024-04-24 15:04:13 -05:00
Elijah DeLee
c11ff49a56 fixup syntax 2024-04-24 14:48:02 -05:00
Elijah DeLee
51bcf82cf4 include x-request-id header in perf log if exists 2024-04-24 13:51:42 -05:00
19 changed files with 179 additions and 136 deletions

75
.github/workflows/e2e_test.yml vendored Normal file
View File

@@ -0,0 +1,75 @@
---
name: E2E Tests
env:
LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
on:
pull_request_target:
types: [labeled]
jobs:
e2e-test:
if: contains(github.event.pull_request.labels.*.name, 'qe:e2e')
runs-on: ubuntu-latest
timeout-minutes: 40
permissions:
packages: write
contents: read
strategy:
matrix:
job: [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24]
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/run_awx_devel
id: awx
with:
build-ui: true
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Pull awx_cypress_base image
run: |
docker pull quay.io/awx/awx_cypress_base:latest
- name: Checkout test project
uses: actions/checkout@v3
with:
repository: ${{ github.repository_owner }}/tower-qa
ssh-key: ${{ secrets.QA_REPO_KEY }}
path: tower-qa
ref: devel
- name: Build cypress
run: |
cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
docker build -t awx-pf-tests .
- name: Run E2E tests
env:
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
run: |
export COMMIT_INFO_BRANCH=$GITHUB_HEAD_REF
export COMMIT_INFO_AUTHOR=$GITHUB_ACTOR
export COMMIT_INFO_SHA=$GITHUB_SHA
export COMMIT_INFO_REMOTE=$GITHUB_REPOSITORY_OWNER
cd ${{ secrets.E2E_PROJECT }}/ui-tests/awx-pf-tests
AWX_IP=${{ steps.awx.outputs.ip }}
printenv > .env
echo "Executing tests:"
docker run \
--network '_sources_default' \
--ipc=host \
--env-file=.env \
-e CYPRESS_baseUrl="https://$AWX_IP:8043" \
-e CYPRESS_AWX_E2E_USERNAME=admin \
-e CYPRESS_AWX_E2E_PASSWORD='password' \
-e COMMAND="npm run cypress-concurrently-gha" \
-v /dev/shm:/dev/shm \
-v $PWD:/e2e \
-w /e2e \
awx-pf-tests run --project .
- uses: ./.github/actions/upload_awx_devel_logs
if: always()
with:
log-filename: e2e-${{ matrix.job }}.log

View File

@@ -61,10 +61,6 @@ class StringListBooleanField(ListField):
def to_representation(self, value):
try:
if isinstance(value, str):
# https://github.com/encode/django-rest-framework/commit/a180bde0fd965915718b070932418cabc831cee1
# DRF changed truthy and falsy lists to be capitalized
value = value.lower()
if isinstance(value, (list, tuple)):
return super(StringListBooleanField, self).to_representation(value)
elif value in BooleanField.TRUE_VALUES:
@@ -82,8 +78,6 @@ class StringListBooleanField(ListField):
def to_internal_value(self, data):
try:
if isinstance(data, str):
data = data.lower()
if isinstance(data, (list, tuple)):
return super(StringListBooleanField, self).to_internal_value(data)
elif data in BooleanField.TRUE_VALUES:

View File

@@ -58,7 +58,7 @@ class TimingMiddleware(threading.local, MiddlewareMixin):
response['X-API-Profile-File'] = self.prof.stop()
perf_logger.debug(
f'request: {request}, response_time: {response["X-API-Total-Time"]}',
extra=dict(python_objects=dict(request=request, response=response, X_API_TOTAL_TIME=response["X-API-Total-Time"])),
extra=dict(python_objects=dict(request=request, response=response, X_API_TOTAL_TIME=response["X-API-Total-Time"], x_request_id=request.get('x-request-id', 'not-set'))),
)
return response

View File

@@ -275,12 +275,7 @@ def setup_managed_role_definitions(apps, schema_editor):
"""
Idepotent method to create or sync the managed role definitions
"""
to_create = {
'object_admin': '{cls.__name__} Admin',
'org_admin': 'Organization Admin',
'org_children': 'Organization {cls.__name__} Admin',
'special': '{cls.__name__} {action}',
}
to_create = settings.ANSIBLE_BASE_ROLE_PRECREATE
ContentType = apps.get_model('contenttypes', 'ContentType')
Permission = apps.get_model('dab_rbac', 'DABPermission')

View File

@@ -10,9 +10,6 @@ import re
# django-rest-framework
from rest_framework.serializers import ValidationError
# crum to impersonate users
from crum import impersonate
# Django
from django.db import models, transaction, connection
from django.db.models.signals import m2m_changed
@@ -556,22 +553,17 @@ def get_role_definition(role):
return
f = obj._meta.get_field(role.role_field)
action_name = f.name.rsplit("_", 1)[0]
model_print = type(obj).__name__
rd_name = f'{model_print} {action_name.title()} Compat'
rd_name = f'{type(obj).__name__} {action_name.title()} Compat'
perm_list = get_role_codenames(role)
defaults = {
'content_type_id': role.content_type_id,
'description': f'Has {action_name.title()} permission to {model_print} for backwards API compatibility',
}
with impersonate(None):
try:
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
except ValidationError:
# This is a tricky case - practically speaking, users should not be allowed to create team roles
# or roles that include the team member permission.
# If we need to create this for compatibility purposes then we will create it as a managed non-editable role
defaults['managed'] = True
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
defaults = {'content_type_id': role.content_type_id}
try:
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
except ValidationError:
# This is a tricky case - practically speaking, users should not be allowed to create team roles
# or roles that include the team member permission.
# If we need to create this for compatibility purposes then we will create it as a managed non-editable role
defaults['managed'] = True
rd, created = RoleDefinition.objects.get_or_create(name=rd_name, permissions=perm_list, defaults=defaults)
return rd

View File

@@ -30,7 +30,7 @@ def test_idempotent_credential_type_setup():
@pytest.mark.django_db
def test_create_user_credential_via_credentials_list(post, get, alice, credentialtype_ssh, setup_managed_roles):
def test_create_user_credential_via_credentials_list(post, get, alice, credentialtype_ssh):
params = {
'credential_type': 1,
'inputs': {'username': 'someusername'},
@@ -81,7 +81,7 @@ def test_credential_validation_error_with_multiple_owner_fields(post, admin, ali
@pytest.mark.django_db
def test_create_user_credential_via_user_credentials_list(post, get, alice, credentialtype_ssh, setup_managed_roles):
def test_create_user_credential_via_user_credentials_list(post, get, alice, credentialtype_ssh):
params = {
'credential_type': 1,
'inputs': {'username': 'someusername'},

View File

@@ -16,8 +16,6 @@ from django.db.backends.sqlite3.base import SQLiteCursorWrapper
from django.db.models.signals import post_migrate
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
# AWX
from awx.main.models.projects import Project
from awx.main.models.ha import Instance
@@ -92,12 +90,6 @@ def deploy_jobtemplate(project, inventory, credential):
return jt
@pytest.fixture
def setup_managed_roles():
"Run the migration script to pre-create managed role definitions"
setup_managed_role_definitions(apps, None)
@pytest.fixture
def team(organization):
return organization.teams.create(name='test-team')

View File

@@ -0,0 +1,10 @@
import pytest
from django.apps import apps
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
@pytest.fixture
def managed_roles():
"Run the migration script to pre-create managed role definitions"
setup_managed_role_definitions(apps, None)

View File

@@ -0,0 +1,45 @@
import pytest
from django.apps import apps
from django.test.utils import override_settings
from awx.main.migrations._dab_rbac import setup_managed_role_definitions
from ansible_base.rbac.models import RoleDefinition
INVENTORY_OBJ_PERMISSIONS = ['view_inventory', 'adhoc_inventory', 'use_inventory', 'change_inventory', 'delete_inventory', 'update_inventory']
@pytest.mark.django_db
def test_managed_definitions_precreate():
with override_settings(
ANSIBLE_BASE_ROLE_PRECREATE={
'object_admin': '{cls._meta.model_name}-admin',
'org_admin': 'organization-admin',
'org_children': 'organization-{cls._meta.model_name}-admin',
'special': '{cls._meta.model_name}-{action}',
}
):
setup_managed_role_definitions(apps, None)
rd = RoleDefinition.objects.get(name='inventory-admin')
assert rd.managed is True
# add permissions do not go in the object-level admin
assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)
# test org-level object admin permissions
rd = RoleDefinition.objects.get(name='organization-inventory-admin')
assert rd.managed is True
assert set(rd.permissions.values_list('codename', flat=True)) == set(['add_inventory', 'view_organization'] + INVENTORY_OBJ_PERMISSIONS)
@pytest.mark.django_db
def test_managed_definitions_custom_obj_admin_name():
with override_settings(
ANSIBLE_BASE_ROLE_PRECREATE={
'object_admin': 'foo-{cls._meta.model_name}-foo',
}
):
setup_managed_role_definitions(apps, None)
rd = RoleDefinition.objects.get(name='foo-inventory-foo')
assert rd.managed is True
# add permissions do not go in the object-level admin
assert set(rd.permissions.values_list('codename', flat=True)) == set(INVENTORY_OBJ_PERMISSIONS)

View File

@@ -10,7 +10,7 @@ from ansible_base.rbac.models import RoleDefinition
@pytest.mark.django_db
def test_managed_roles_created(setup_managed_roles):
def test_managed_roles_created(managed_roles):
"Managed RoleDefinitions are created in post_migration signal, we expect to see them here"
for cls in (JobTemplate, Inventory):
ct = ContentType.objects.get_for_model(cls)
@@ -22,7 +22,7 @@ def test_managed_roles_created(setup_managed_roles):
@pytest.mark.django_db
def test_custom_read_role(admin_user, post, setup_managed_roles):
def test_custom_read_role(admin_user, post, managed_roles):
rd_url = django_reverse('roledefinition-list')
resp = post(
url=rd_url, data={"name": "read role made for test", "content_type": "awx.inventory", "permissions": ['view_inventory']}, user=admin_user, expect=201
@@ -40,7 +40,7 @@ def test_custom_system_roles_prohibited(admin_user, post):
@pytest.mark.django_db
def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post, setup_managed_roles):
def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post, managed_roles):
"Alice can not see rando, and so can not give them a role assignment"
rd = RoleDefinition.objects.get(name='Inventory Admin')
rd.give_permission(alice, inventory)
@@ -51,7 +51,7 @@ def test_assignment_to_invisible_user(admin_user, alice, rando, inventory, post,
@pytest.mark.django_db
def test_assign_managed_role(admin_user, alice, rando, inventory, post, setup_managed_roles, organization):
def test_assign_managed_role(admin_user, alice, rando, inventory, post, managed_roles, organization):
rd = RoleDefinition.objects.get(name='Inventory Admin')
rd.give_permission(alice, inventory)
# When alice and rando are members of the same org, they can see each other
@@ -78,7 +78,7 @@ def test_assign_custom_delete_role(admin_user, rando, inventory, delete, patch):
@pytest.mark.django_db
def test_assign_custom_add_role(admin_user, rando, organization, post, setup_managed_roles):
def test_assign_custom_add_role(admin_user, rando, organization, post, managed_roles):
rd, _ = RoleDefinition.objects.get_or_create(
name='inventory-add', permissions=['add_inventory', 'view_organization'], content_type=ContentType.objects.get_for_model(Organization)
)

View File

@@ -2,15 +2,11 @@ from unittest import mock
import pytest
from django.contrib.contenttypes.models import ContentType
from crum import impersonate
from awx.main.models.rbac import get_role_from_object_role, give_creator_permissions
from awx.main.models import User, Organization, WorkflowJobTemplate, WorkflowJobTemplateNode, Team
from awx.api.versioning import reverse
from ansible_base.rbac.models import RoleUserAssignment, RoleDefinition
from ansible_base.rbac.models import RoleUserAssignment
@pytest.mark.django_db
@@ -18,7 +14,7 @@ from ansible_base.rbac.models import RoleUserAssignment, RoleDefinition
'role_name',
['execution_environment_admin_role', 'project_admin_role', 'admin_role', 'auditor_role', 'read_role', 'execute_role', 'notification_admin_role'],
)
def test_round_trip_roles(organization, rando, role_name, setup_managed_roles):
def test_round_trip_roles(organization, rando, role_name, managed_roles):
"""
Make an assignment with the old-style role,
get the equivelent new role
@@ -32,39 +28,7 @@ def test_round_trip_roles(organization, rando, role_name, setup_managed_roles):
@pytest.mark.django_db
def test_role_naming(setup_managed_roles):
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='dmin')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Admin'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_action_role_naming(setup_managed_roles):
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='ecute')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Execute'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_compat_role_naming(setup_managed_roles, job_template, rando, alice):
with impersonate(alice):
job_template.read_role.members.add(rando)
qs = RoleDefinition.objects.filter(content_type=ContentType.objects.get(model='jobtemplate'), name__endswith='ompat')
assert qs.count() == 1 # sanity
rd = qs.first()
assert rd.name == 'JobTemplate Read Compat'
assert rd.description
assert rd.created_by is None
@pytest.mark.django_db
def test_organization_level_permissions(organization, inventory, setup_managed_roles):
def test_organization_level_permissions(organization, inventory, managed_roles):
u1 = User.objects.create(username='alice')
u2 = User.objects.create(username='bob')
@@ -94,14 +58,14 @@ def test_organization_level_permissions(organization, inventory, setup_managed_r
@pytest.mark.django_db
def test_organization_execute_role(organization, rando, setup_managed_roles):
def test_organization_execute_role(organization, rando, managed_roles):
organization.execute_role.members.add(rando)
assert rando in organization.execute_role
assert set(Organization.accessible_objects(rando, 'execute_role')) == set([organization])
@pytest.mark.django_db
def test_workflow_approval_list(get, post, admin_user, setup_managed_roles):
def test_workflow_approval_list(get, post, admin_user, managed_roles):
workflow_job_template = WorkflowJobTemplate.objects.create()
approval_node = WorkflowJobTemplateNode.objects.create(workflow_job_template=workflow_job_template)
url = reverse('api:workflow_job_template_node_create_approval', kwargs={'pk': approval_node.pk, 'version': 'v2'})
@@ -115,14 +79,14 @@ def test_workflow_approval_list(get, post, admin_user, setup_managed_roles):
@pytest.mark.django_db
def test_creator_permission(rando, admin_user, inventory, setup_managed_roles):
def test_creator_permission(rando, admin_user, inventory, managed_roles):
give_creator_permissions(rando, inventory)
assert rando in inventory.admin_role
assert rando in inventory.admin_role.members.all()
@pytest.mark.django_db
def test_team_team_read_role(rando, team, admin_user, post, setup_managed_roles):
def test_team_team_read_role(rando, team, admin_user, post, managed_roles):
orgs = [Organization.objects.create(name=f'foo-{i}') for i in range(2)]
teams = [Team.objects.create(name=f'foo-{i}', organization=orgs[i]) for i in range(2)]
teams[1].member_role.members.add(rando)

View File

@@ -165,7 +165,7 @@ class TestOrphanJobTemplate:
@pytest.mark.django_db
@pytest.mark.job_permissions
def test_job_template_creator_access(project, organization, rando, post, setup_managed_roles):
def test_job_template_creator_access(project, organization, rando, post):
project.use_role.members.add(rando)
response = post(
url=reverse('api:job_template_list'),

View File

@@ -1148,8 +1148,13 @@ ANSIBLE_BASE_CUSTOM_VIEW_PARENT = 'awx.api.generics.APIView'
# Settings for the ansible_base RBAC system
# This has been moved to data migration code
ANSIBLE_BASE_ROLE_PRECREATE = {}
# Only used internally, names of the managed RoleDefinitions to create
ANSIBLE_BASE_ROLE_PRECREATE = {
'object_admin': '{cls.__name__} Admin',
'org_admin': 'Organization Admin',
'org_children': 'Organization {cls.__name__} Admin',
'special': '{cls.__name__} {action}',
}
# Name for auto-created roles that give users permissions to what they create
ANSIBLE_BASE_ROLE_CREATOR_NAME = '{cls.__name__} Creator'

View File

@@ -1038,10 +1038,7 @@ class ControllerAPIModule(ControllerModule):
# Grab our start time to compare against for the timeout
start = time.time()
result = self.get_endpoint(url)
wait_on_field = 'event_processing_finished'
if wait_on_field not in result['json']:
wait_on_field = 'finished'
while not result['json'][wait_on_field]:
while not result['json']['finished']:
# If we are past our time out fail with a message
if timeout and timeout < time.time() - start:
# Account for Legacy messages

View File

@@ -234,7 +234,7 @@ class ApiV2(base.Base):
return endpoint.get(**{identifier: value}, all_pages=True)
def export_assets(self, **kwargs):
self._cache = page.PageCache(self.connection)
self._cache = page.PageCache()
# If no resource kwargs are explicitly used, export everything.
all_resources = all(kwargs.get(resource) is None for resource in EXPORTABLE_RESOURCES)
@@ -335,7 +335,7 @@ class ApiV2(base.Base):
if name == 'roles':
indexed_roles = defaultdict(list)
for role in S:
if role.get('content_object') is None:
if 'content_object' not in role:
continue
indexed_roles[role['content_object']['type']].append(role)
self._roles.append((_page, indexed_roles))
@@ -411,7 +411,7 @@ class ApiV2(base.Base):
# FIXME: deal with pruning existing relations that do not match the import set
def import_assets(self, data):
self._cache = page.PageCache(self.connection)
self._cache = page.PageCache()
self._related = []
self._roles = []
@@ -420,8 +420,11 @@ class ApiV2(base.Base):
for resource in self._dependent_resources():
endpoint = getattr(self, resource)
# Load up existing objects, so that we can try to update or link to them
self._cache.get_page(endpoint)
imported = self._import_list(endpoint, data.get(resource) or [])
changed = changed or imported
# FIXME: should we delete existing unpatched assets?
self._assign_related()
self._assign_membership()

View File

@@ -11,7 +11,6 @@ from awxkit.utils import PseudoNamespace, is_relative_endpoint, are_same_endpoin
from awxkit.api import utils
from awxkit.api.client import Connection
from awxkit.api.registry import URLRegistry
from awxkit.api.resources import resources
from awxkit.config import config
import awxkit.exceptions as exc
@@ -494,11 +493,10 @@ class TentativePage(str):
class PageCache(object):
def __init__(self, connection=None):
def __init__(self):
self.options = {}
self.pages_by_url = {}
self.pages_by_natural_key = {}
self.connection = connection or Connection(config.base_url, not config.assume_untrusted)
def get_options(self, page):
url = page.endpoint if isinstance(page, Page) else str(page)
@@ -552,31 +550,7 @@ class PageCache(object):
return self.set_page(page)
def get_by_natural_key(self, natural_key):
page = self.pages_by_natural_key.get(utils.freeze(natural_key))
if page is None:
# We need some way to get ahold of the top-level resource
# list endpoint from the natural_key type. The resources
# object more or less has that for each of the detail
# views. Just chop off the /<id>/ bit.
endpoint = getattr(resources, natural_key['type'], None)
if endpoint is None:
return
endpoint = ''.join([endpoint.rsplit('/', 2)[0], '/'])
page_type = get_registered_page(endpoint)
kwargs = {}
for k, v in natural_key.items():
if isinstance(v, str) and k != 'type':
kwargs[k] = v
# Do a filtered query against the list endpoint, usually
# with the name of the object but sometimes more.
list_page = page_type(self.connection, endpoint=endpoint).get(all_pages=True, **kwargs)
if 'results' in list_page:
for p in list_page.results:
self.set_page(p)
page = self.pages_by_natural_key.get(utils.freeze(natural_key))
log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), page)
if page:
return self.get_page(page)
endpoint = self.pages_by_natural_key.get(utils.freeze(natural_key))
log.debug("get_by_natural_key: %s, endpoint: %s", repr(natural_key), endpoint)
if endpoint:
return self.get_page(endpoint)

View File

@@ -25,7 +25,7 @@ django-pglocks
django-radius
django-solo
django-split-settings==1.0.0 # We hit a strange issue where the release process errored when upgrading past 1.0.0 see UPGRADE BLOCKERS
djangorestframework>=3.15.0
djangorestframework
djangorestframework-yaml
filelock
GitPython>=3.1.37 # CVE-2023-41040

View File

@@ -167,7 +167,7 @@ django-split-settings==1.0.0
# via
# -r /awx_devel/requirements/requirements.in
# django-ansible-base
djangorestframework==3.15.1
djangorestframework==3.14.0
# via
# -r /awx_devel/requirements/requirements.in
# django-ansible-base
@@ -387,6 +387,7 @@ python3-openid==3.2.0
# via -r /awx_devel/requirements/requirements_git.txt
pytz==2024.1
# via
# djangorestframework
# irc
# tempora
pyyaml==6.0.1

View File

@@ -36,8 +36,6 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
{% endif %}
nss \
openldap-devel \
# pin to older openssl, see jira AAP-23449
openssl-3.0.7 \
patch \
postgresql \
postgresql-devel \
@@ -122,8 +120,6 @@ RUN dnf -y update && dnf install -y 'dnf-command(config-manager)' && \
krb5-workstation \
nginx \
"openldap >= 2.6.2-3" \
# pin to older openssl, see jira AAP-23449
openssl-3.0.7 \
postgresql \
python3.11 \
"python3.11-devel" \