Compare commits

..

2 Commits

Author SHA1 Message Date
Peter Braun
ed1b5c5519 exclude password fields from skipping if not defined 2026-02-17 12:19:43 +01:00
Peter Braun
d2e51c4124 do not add optional survey fields with empty strings that are not backed by extra_vars 2026-02-17 10:13:33 +01:00
87 changed files with 694 additions and 4419 deletions

View File

@@ -24,7 +24,7 @@ in as the first entry for your PR title.
##### STEPS TO REPRODUCE AND EXTRA INFO
##### ADDITIONAL INFORMATION
<!---
Include additional information to help people understand the change here.
For bugs that don't have a linked bug report, a step-by-step reproduction

View File

@@ -4,46 +4,14 @@ env:
LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEV_DOCKER_OWNER: ${{ github.repository_owner }}
COMPOSE_TAG: ${{ github.base_ref || github.ref_name || 'devel' }}
COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
UPSTREAM_REPOSITORY_ID: 91594105
on:
pull_request:
push:
branches:
- devel # needed to publish code coverage post-merge
schedule:
- cron: '0 12,18 * * 1-5'
workflow_dispatch: {}
jobs:
trigger-release-branches:
name: "Dispatch CI to release branches"
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
permissions:
actions: write
steps:
- name: Trigger CI on release_4.6
id: dispatch_release_46
continue-on-error: true
run: gh workflow run ci.yml --ref release_4.6
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
- name: Trigger CI on stable-2.6
id: dispatch_stable_26
continue-on-error: true
run: gh workflow run ci.yml --ref stable-2.6
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
- name: Check dispatch results
if: steps.dispatch_release_46.outcome == 'failure' || steps.dispatch_stable_26.outcome == 'failure'
run: |
echo "One or more dispatches failed:"
echo " release_4.6: ${{ steps.dispatch_release_46.outcome }}"
echo " stable-2.6: ${{ steps.dispatch_stable_26.outcome }}"
exit 1
common-tests:
name: ${{ matrix.tests.name }}
runs-on: ubuntu-latest
@@ -94,11 +62,7 @@ jobs:
run: |
if [ -f "reports/coverage.xml" ]; then
sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' reports/coverage.xml
echo "Injected PR number ${{ github.event.pull_request.number }} into reports/coverage.xml"
fi
if [ -f "awxkit/coverage.xml" ]; then
sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' awxkit/coverage.xml
echo "Injected PR number ${{ github.event.pull_request.number }} into awxkit/coverage.xml"
echo "Injected PR number ${{ github.event.pull_request.number }} into coverage.xml"
fi
- name: Upload test coverage to Codecov
@@ -145,9 +109,7 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.tests.name }}-artifacts
path: |
reports/coverage.xml
awxkit/coverage.xml
path: reports/coverage.xml
retention-days: 5
- name: >-
@@ -160,7 +122,7 @@ jobs:
&& github.event_name == 'push'
&& env.UPSTREAM_REPOSITORY_ID == github.repository_id
&& github.ref_name == github.event.repository.default_branch
uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
uses: ansible/gh-action-record-test-results@cd5956ead39ec66351d0779470c8cff9638dd2b8
with:
aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
http-auth-password: >-
@@ -334,7 +296,7 @@ jobs:
&& github.event_name == 'push'
&& env.UPSTREAM_REPOSITORY_ID == github.repository_id
&& github.ref_name == github.event.repository.default_branch
uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
uses: ansible/gh-action-record-test-results@cd5956ead39ec66351d0779470c8cff9638dd2b8
with:
aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
http-auth-password: >-

View File

@@ -131,14 +131,8 @@ class LoggedLoginView(auth_views.LoginView):
class LoggedLogoutView(auth_views.LogoutView):
# Override http_method_names to allow GET requests (Django 5.2+ defaults to POST only)
http_method_names = ["get", "post", "options"]
success_url_allowed_hosts = set(settings.LOGOUT_ALLOWED_HOSTS.split(",")) if settings.LOGOUT_ALLOWED_HOSTS else set()
def get(self, request, *args, **kwargs):
"""Handle GET requests for logout (for backward compatibility)."""
return self.post(request, *args, **kwargs)
def dispatch(self, request, *args, **kwargs):
if is_proxied_request():
# 1) We intentionally don't obey ?next= here, just always redirect to platform login

View File

@@ -55,7 +55,6 @@ from wsgiref.util import FileWrapper
from drf_spectacular.utils import extend_schema_view, extend_schema
# django-ansible-base
from ansible_base.lib.utils.requests import get_remote_hosts
from ansible_base.rbac.models import RoleEvaluation
from ansible_base.lib.utils.schema import extend_schema_if_available
@@ -98,6 +97,7 @@ from awx.main.utils import (
from awx.main.utils.encryption import encrypt_value
from awx.main.utils.filters import SmartFilter
from awx.main.utils.plugins import compute_cloud_inventory_sources
from awx.main.utils.proxy import get_first_remote_host_from_headers
from awx.main.utils.common import memoize
from awx.main.redact import UriCleaner
from awx.api.permissions import (
@@ -2877,7 +2877,8 @@ class JobTemplateCallback(GenericAPIView):
host for the current request.
"""
# Find the list of remote host names/IPs to check.
remote_hosts = set(get_remote_hosts(self.request))
# Only consider the first entry from each header (for comma-separated values like X-Forwarded-For)
remote_hosts = get_first_remote_host_from_headers(self.request, settings.REMOTE_HOST_HEADERS)
# Add the reverse lookup of IP addresses.
for rh in list(remote_hosts):
try:

View File

@@ -11,7 +11,6 @@ __all__ = [
'CAN_CANCEL',
'ACTIVE_STATES',
'STANDARD_INVENTORY_UPDATE_ENV',
'OIDC_CREDENTIAL_TYPE_NAMESPACES',
]
PRIVILEGE_ESCALATION_METHODS = [
@@ -141,6 +140,3 @@ org_role_to_permission = {
'execution_environment_admin_role': 'add_executionenvironment',
'auditor_role': 'view_project', # TODO: also doesnt really work
}
# OIDC credential type namespaces for feature flag filtering
OIDC_CREDENTIAL_TYPE_NAMESPACES = ['hashivault-kv-oidc', 'hashivault-ssh-oidc']

View File

@@ -27,10 +27,6 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
"pool_kwargs": {
"min_workers": settings.JOB_EVENT_WORKERS,
"max_workers": max_workers,
# This must be less than max_workers to make sense, which is usually 4
# With reserve of 1, after a burst of tasks, load needs to down to 4-1=3
# before we return to min_workers
"scaledown_reserve": 1,
},
"main_kwargs": {"node_id": settings.CLUSTER_HOST_ID},
"process_manager_cls": "ForkServerManager",

View File

@@ -77,13 +77,13 @@ class CallbackBrokerWorker:
MAX_RETRIES = 2
INDIVIDUAL_EVENT_RETRIES = 3
last_stats = time.time()
last_flush = time.time()
total = 0
last_event = ''
prof = None
def __init__(self):
self.last_stats = time.time()
self.last_flush = time.time()
self.buff = {}
self.redis = get_redis_client()
self.subsystem_metrics = s_metrics.CallbackReceiverMetrics(auto_pipe_execute=False)

View File

@@ -428,9 +428,6 @@ class CredentialInputField(JSONSchemaField):
# determine the defined fields for the associated credential type
properties = {}
for field in model_instance.credential_type.inputs.get('fields', []):
# Prevent users from providing values for internally resolved fields
if 'internal' in field:
continue
field = field.copy()
properties[field['id']] = field
if field.get('choices', []):
@@ -569,7 +566,6 @@ class CredentialTypeInputField(JSONSchemaField):
},
'label': {'type': 'string'},
'help_text': {'type': 'string'},
'internal': {'type': 'boolean'},
'multiline': {'type': 'boolean'},
'secret': {'type': 'boolean'},
'ask_at_runtime': {'type': 'boolean'},

View File

@@ -1,29 +0,0 @@
# Generated by Django 5.2.8 on 2026-02-20 03:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0204_squashed_deletions'),
]
operations = [
migrations.AlterModelOptions(
name='instancegroup',
options={
'default_permissions': ('change', 'delete', 'view'),
'ordering': ('pk',),
'permissions': [('use_instancegroup', 'Can use instance group in a preference list of a resource')],
},
),
migrations.AlterModelOptions(
name='workflowjobnode',
options={'ordering': ('pk',)},
),
migrations.AlterModelOptions(
name='workflowjobtemplatenode',
options={'ordering': ('pk',)},
),
]

View File

@@ -28,7 +28,6 @@ from rest_framework.serializers import ValidationError as DRFValidationError
from ansible_base.lib.utils.db import advisory_lock
# AWX
from awx.main.constants import OIDC_CREDENTIAL_TYPE_NAMESPACES
from awx.api.versioning import reverse
from awx.main.fields import (
ImplicitRoleField,
@@ -243,29 +242,6 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
needed.append('vault_password')
return needed
@functools.cached_property
def context(self):
"""
Property for storing runtime context during credential resolution.
The context is a dict keyed by CredentialInputSource PK, where each value
is a dict of runtime fields for that input source. Example::
{
<input_source_pk>: {
"workload_identity_token": "<jwt_token>"
},
<another_input_source_pk>: {
"workload_identity_token": "<different_jwt_token>"
},
}
This structure allows each input source to have its own set of runtime
values, avoiding conflicts when a credential has multiple input sources
with different configurations (e.g., different JWT audiences).
"""
return {}
@cached_property
def dynamic_input_fields(self):
# if the credential is not yet saved we can't access the input_sources
@@ -391,7 +367,7 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
def _get_dynamic_input(self, field_name):
for input_source in self.input_sources.all():
if input_source.input_field_name == field_name:
return input_source.get_input_value(context=self.context)
return input_source.get_input_value()
else:
raise ValueError('{} is not a dynamic input field'.format(field_name))
@@ -459,15 +435,13 @@ class CredentialType(CommonModelNameNotUnique):
def from_db(cls, db, field_names, values):
instance = super(CredentialType, cls).from_db(db, field_names, values)
if instance.managed and instance.namespace and instance.kind != "external":
native = ManagedCredentialType.registry.get(instance.namespace)
if native:
instance.inputs = native.inputs
instance.injectors = native.injectors
instance.custom_injectors = getattr(native, 'custom_injectors', None)
native = ManagedCredentialType.registry[instance.namespace]
instance.inputs = native.inputs
instance.injectors = native.injectors
instance.custom_injectors = getattr(native, 'custom_injectors', None)
elif instance.namespace and instance.kind == "external":
native = ManagedCredentialType.registry.get(instance.namespace)
if native:
instance.inputs = native.inputs
native = ManagedCredentialType.registry[instance.namespace]
instance.inputs = native.inputs
return instance
@@ -648,15 +622,7 @@ class CredentialInputSource(PrimordialModel):
raise ValidationError(_('Input field must be defined on target credential (options are {}).'.format(', '.join(sorted(defined_fields)))))
return self.input_field_name
def get_input_value(self, context: dict | None = None):
"""
Retrieve the value from the external credential backend.
Args:
context: Optional runtime context dict passed from the target credential.
"""
if context is None:
context = {}
def get_input_value(self):
backend = self.source_credential.credential_type.plugin.backend
backend_kwargs = {}
for field_name, value in self.source_credential.inputs.items():
@@ -667,17 +633,6 @@ class CredentialInputSource(PrimordialModel):
backend_kwargs.update(self.metadata)
# Resolve internal fields from the per-input-source context.
# The context dict is keyed by input source PK, e.g.:
# {42: {"workload_identity_token": "eyJ..."}, 43: {"workload_identity_token": "eyX..."}}
# This allows each input source to carry its own runtime values.
input_source_context = context.get(self.pk, {})
for field in self.source_credential.credential_type.inputs.get('fields', []):
if field.get('internal'):
value = input_source_context.get(field['id'])
if value is not None:
backend_kwargs[field['id']] = value
with set_environ(**settings.AWX_TASK_ENV):
return backend(**backend_kwargs)
@@ -686,20 +641,13 @@ class CredentialInputSource(PrimordialModel):
return reverse(view_name, kwargs={'pk': self.pk}, request=request)
def _is_oidc_namespace_disabled(ns):
"""Check if a credential namespace should be skipped based on the OIDC feature flag."""
return ns in OIDC_CREDENTIAL_TYPE_NAMESPACES and not getattr(settings, 'FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED', False)
def load_credentials():
awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')}
supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')}
plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points}
for ns, ep in plugin_entry_points.items():
if _is_oidc_namespace_disabled(ns):
continue
cred_plugin = ep.load()
if not hasattr(cred_plugin, 'inputs'):
setattr(cred_plugin, 'inputs', {})
@@ -718,8 +666,5 @@ def load_credentials():
credential_plugins = {}
for ns, ep in credential_plugins.items():
if _is_oidc_namespace_disabled(ns):
continue
plugin = ep.load()
CredentialType.load_plugin(ns, plugin)

View File

@@ -485,7 +485,6 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMi
class Meta:
app_label = 'main'
ordering = ('pk',)
permissions = [('use_instancegroup', 'Can use instance group in a preference list of a resource')]
# Since this has no direct organization field only superuser can add, so remove add permission
default_permissions = ('change', 'delete', 'view')

View File

@@ -845,21 +845,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
def get_notification_friendly_name(self):
return "Job"
def get_source_hosts_for_constructed_inventory(self):
"""Return a QuerySet of the source (input inventory) hosts for a constructed inventory.
Constructed inventory hosts have an instance_id pointing to the real
host in the input inventory. This resolves those references and returns
a proper QuerySet (never a list), suitable for use with finish_fact_cache.
"""
Host = JobHostSummary._meta.get_field('host').related_model
if not self.inventory_id:
return Host.objects.none()
id_field = Host._meta.get_field('id')
return Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field))).only(
*HOST_FACTS_FIELDS
)
def get_hosts_for_fact_cache(self):
"""
Builds the queryset to use for writing or finalizing the fact cache
@@ -867,15 +852,17 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
For constructed inventories, that means the original (input inventory) hosts
when slicing, that means only returning hosts in that slice
"""
Host = JobHostSummary._meta.get_field('host').related_model
if not self.inventory_id:
Host = JobHostSummary._meta.get_field('host').related_model
return Host.objects.none()
if self.inventory.kind == 'constructed':
host_qs = self.get_source_hosts_for_constructed_inventory()
id_field = Host._meta.get_field('id')
host_qs = Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field)))
else:
host_qs = self.inventory.hosts.only(*HOST_FACTS_FIELDS)
host_qs = self.inventory.hosts
host_qs = host_qs.only(*HOST_FACTS_FIELDS)
host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count)
return host_qs

View File

@@ -200,7 +200,6 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
indexes = [
models.Index(fields=['identifier']),
]
ordering = ('pk',)
def get_absolute_url(self, request=None):
return reverse('api:workflow_job_template_node_detail', kwargs={'pk': self.pk}, request=request)
@@ -287,7 +286,6 @@ class WorkflowJobNode(WorkflowNodeBase):
models.Index(fields=["identifier", "workflow_job"]),
models.Index(fields=['identifier']),
]
ordering = ('pk',)
@property
def event_processing_finished(self):
@@ -918,17 +916,6 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
ScheduleWorkflowManager().schedule()
return reverse('api:workflow_approval_deny', kwargs={'pk': self.pk}, request=request)
def cancel(self, job_explanation=None, is_chain=False):
# WorkflowApprovals have no dispatcher process (they wait for human
# input) and are excluded from TaskManager processing, so the base
# cancel() would only set cancel_flag without ever transitioning the
# status. We call super() for the flag, then transition directly.
has_already_canceled = bool(self.status == 'canceled')
super().cancel(job_explanation=job_explanation, is_chain=is_chain)
if self.status != 'canceled' and not has_already_canceled:
self.status = 'canceled'
self.save(update_fields=['status'])
def signal_start(self, **kwargs):
can_start = super(WorkflowApproval, self).signal_start(**kwargs)
self.started = self.created

View File

@@ -19,8 +19,13 @@ class ActivityStreamRegistrar(object):
pre_delete.connect(activity_stream_delete, sender=model, dispatch_uid=str(self.__class__) + str(model) + "_delete")
for m2mfield in model._meta.many_to_many:
m2m_attr = getattr(model, m2mfield.name)
m2m_changed.connect(activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate")
try:
m2m_attr = getattr(model, m2mfield.name)
m2m_changed.connect(
activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate"
)
except AttributeError:
pass
def disconnect(self, model):
if model in self.models:

View File

@@ -48,6 +48,11 @@ class SimpleDAG(object):
'''
self.node_to_edges_by_label = dict()
def __contains__(self, obj):
if self.node['node_object'] in self.node_obj_to_node_index:
return True
return False
def __len__(self):
return len(self.nodes)

View File

@@ -122,11 +122,8 @@ class WorkflowDAG(SimpleDAG):
if not job:
continue
elif job.can_cancel:
cancel_finished = False
job.cancel()
# If the job is not yet in a terminal state after .cancel(),
# the TaskManager still needs to process it.
if job.status not in ('successful', 'failed', 'canceled', 'error'):
cancel_finished = False
return cancel_finished
def is_workflow_done(self):

View File

@@ -196,10 +196,6 @@ class WorkflowManager(TaskBase):
workflow_job.start_args = '' # blank field to remove encrypted passwords
workflow_job.save(update_fields=['status', 'start_args'])
status_changed = True
else:
# Speed-up: schedule the task manager so it can process the
# canceled pending jobs without waiting for the next cycle.
ScheduleTaskManager().schedule()
else:
dnr_nodes = dag.mark_dnr_nodes()
WorkflowJobNode.objects.bulk_update(dnr_nodes, ['do_not_run'])
@@ -447,29 +443,17 @@ class TaskManager(TaskBase):
self.controlplane_ig = self.tm_models.instance_groups.controlplane_ig
def process_job_dep_failures(self, task):
"""If job depends on a job that has failed or been canceled, mark as failed.
Returns True if a dep failure was found, False otherwise.
"""
"""If job depends on a job that has failed, mark as failed and handle misc stuff."""
for dep in task.dependent_jobs.all():
# if we detect a failed, error, or canceled dependency, go ahead and fail this task.
if dep.status in ("error", "failed", "canceled"):
# if we detect a failed or error dependency, go ahead and fail this task.
if dep.status in ("error", "failed"):
task.status = 'failed'
if dep.status == 'canceled':
logger.warning(f'Previous task canceled, failing task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
ScheduleWorkflowManager().schedule() # speedup for dependency chains in workflow, on workflow cancel
else:
logger.warning(f'Previous task failed, failing task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
logger.warning(f'Previous task failed task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
task.save(update_fields=['status', 'job_explanation'])
task.websocket_emit_status('failed')
self.pre_start_failed.append(task.id)
@@ -561,17 +545,8 @@ class TaskManager(TaskBase):
logger.warning("Task manager has reached time out while processing pending jobs, exiting loop early")
break
if task.cancel_flag:
logger.debug(f"Canceling pending task {task.log_format} because cancel_flag is set")
task.status = 'canceled'
task.job_explanation = gettext_noop("This job was canceled before it started.")
task.save(update_fields=['status', 'job_explanation'])
task.websocket_emit_status('canceled')
self.pre_start_failed.append(task.id)
ScheduleWorkflowManager().schedule()
continue
if self.process_job_dep_failures(task):
has_failed = self.process_job_dep_failures(task)
if has_failed:
continue
blocked_by = self.job_blocked_by(task)

View File

@@ -277,6 +277,7 @@ class RunnerCallback:
def artifacts_handler(self, artifact_dir):
success, query_file_contents = try_load_query_file(artifact_dir)
if success:
self.delay_update(event_queries_processed=False)
collections_info = collect_queries(query_file_contents)
for collection, data in collections_info.items():
version = data['version']
@@ -300,24 +301,6 @@ class RunnerCallback:
else:
logger.warning(f'The file {COLLECTION_FILENAME} unexpectedly did not contain ansible_version')
# Write event_queries_processed and installed_collections directly
# to the DB instead of using delay_update. delay_update defers
# writes until the final job status save, but
# events_processed_hook (called from both the task runner after
# the final save and the callback receiver after the wrapup
# event) needs event_queries_processed=False visible in the DB
# to dispatch save_indirect_host_entries. The field defaults to
# True, so without a direct write the hook would see True and
# skip the dispatch. installed_collections is also written
# directly so it is available if the callback receiver
# dispatches before the final save.
from awx.main.models import Job
db_updates = {'event_queries_processed': False}
if 'installed_collections' in query_file_contents:
db_updates['installed_collections'] = query_file_contents['installed_collections']
Job.objects.filter(id=self.instance.id).update(**db_updates)
self.artifacts_processed = True

View File

@@ -25,8 +25,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
log_data = log_data or {}
log_data['inventory_id'] = inventory_id
log_data['written_ct'] = 0
# Dict mapping host name -> bool (True if a fact file was written)
hosts_cached = {}
hosts_cached = []
# Create the fact_cache directory inside artifacts_dir
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
@@ -38,14 +37,13 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
last_write_time = None
for host in hosts:
hosts_cached.append(host.name)
if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
hosts_cached[host.name] = False
continue # facts are expired - do not write them
filepath = os.path.join(fact_cache_dir, host.name)
if not os.path.realpath(filepath).startswith(fact_cache_dir):
logger.error(f'facts for host {smart_str(host.name)} could not be cached')
hosts_cached[host.name] = False
continue
try:
@@ -53,18 +51,9 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
os.chmod(f.name, 0o600)
json.dump(host.ansible_facts, f)
log_data['written_ct'] += 1
# Backdate the file by 2 seconds so finish_fact_cache can reliably
# distinguish these reference files from files updated by ansible.
# This guarantees fact file mtime < summary file mtime even with
# zipfile's 2-second timestamp rounding during artifact transfer.
mtime = os.path.getmtime(filepath)
backdated = mtime - 2
os.utime(filepath, (backdated, backdated))
last_write_time = backdated
hosts_cached[host.name] = True
last_write_time = os.path.getmtime(filepath)
except IOError:
logger.error(f'facts for host {smart_str(host.name)} could not be cached')
hosts_cached[host.name] = False
continue
# Write summary file directly to the artifacts_dir
@@ -73,6 +62,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
summary_data = {
'last_write_time': last_write_time,
'hosts_cached': hosts_cached,
'written_ct': log_data['written_ct'],
}
with open(summary_file, 'w', encoding='utf-8') as f:
json.dump(summary_data, f, indent=2)
@@ -84,7 +74,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
add_log_data=True,
)
def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, job_created=None, log_data=None):
def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=None):
log_data = log_data or {}
log_data['inventory_id'] = inventory_id
log_data['updated_ct'] = 0
@@ -104,9 +94,8 @@ def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, jo
logger.error(f'Error reading summary file at {summary_path}: {e}')
return
hosts_cached_map = summary.get('hosts_cached', {})
host_names = list(hosts_cached_map.keys())
hosts_cached = host_qs.filter(name__in=host_names).order_by('id').iterator()
host_names = summary.get('hosts_cached', [])
hosts_cached = Host.objects.filter(name__in=host_names).order_by('id').iterator()
# Path where individual fact files were written
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
hosts_to_update = []
@@ -147,35 +136,16 @@ def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, jo
else:
log_data['unmodified_ct'] += 1
else:
# File is missing. Only interpret this as "ansible cleared facts" if
# start_fact_cache actually wrote a file for this host (i.e. the host
# had valid, non-expired facts before the job ran). If no file was
# ever written, the missing file is expected and not a clear signal.
if not hosts_cached_map.get(host.name):
log_data['unmodified_ct'] += 1
continue
# if the file goes missing, ansible removed it (likely via clear_facts)
# if the file goes missing, but the host has not started facts, then we should not clear the facts
if job_created and host.ansible_facts_modified and host.ansible_facts_modified > job_created:
logger.warning(
f'Skipping fact clear for host {smart_str(host.name)} in job {job_id} '
f'inventory {inventory_id}: host ansible_facts_modified '
f'({host.ansible_facts_modified.isoformat()}) is after this job\'s '
f'created time ({job_created.isoformat()}). '
f'A concurrent job likely updated this host\'s facts while this job was running.'
)
log_data['unmodified_ct'] += 1
else:
host.ansible_facts = {}
host.ansible_facts_modified = now()
hosts_to_update.append(host)
logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
log_data['cleared_ct'] += 1
host.ansible_facts = {}
host.ansible_facts_modified = now()
hosts_to_update.append(host)
logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
log_data['cleared_ct'] += 1
if len(hosts_to_update) >= 100:
bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])
hosts_to_update = []
bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])
logger.debug(f'Updated {log_data["updated_ct"]} host facts for inventory {inventory_id} in job {job_id}')

View File

@@ -17,6 +17,7 @@ import urllib.parse as urlparse
# Django
from django.conf import settings
from django.db import transaction
# Shared code for the AWX platform
from awx_plugins.interfaces._temporary_private_container_api import CONTAINER_ROOT, get_incontainer_path
@@ -94,7 +95,6 @@ from flags.state import flag_enabled
# Workload Identity
from ansible_base.lib.workload_identity.controller import AutomationControllerJobScope
from ansible_base.resource_registry.workload_identity_client import get_workload_identity_client
logger = logging.getLogger('awx.main.tasks.jobs')
@@ -104,6 +104,11 @@ def populate_claims_for_workload(unified_job) -> dict:
Extract JWT claims from a Controller workload for the aap_controller_automation_job scope.
"""
# Related objects in the UnifiedJob model, applies to all job types
organization = getattr_dne(unified_job, 'organization')
ujt = getattr_dne(unified_job, 'unified_job_template')
instance_group = getattr_dne(unified_job, 'instance_group')
claims = {
AutomationControllerJobScope.CLAIM_JOB_ID: unified_job.id,
AutomationControllerJobScope.CLAIM_JOB_NAME: unified_job.name,
@@ -158,26 +163,6 @@ def populate_claims_for_workload(unified_job) -> dict:
return claims
def retrieve_workload_identity_jwt(
unified_job: UnifiedJob,
audience: str,
scope: str,
workload_ttl_seconds: int | None = None,
) -> str:
"""Retrieve JWT token from workload claims.
Raises:
RuntimeError: if the workload identity client is not configured.
"""
client = get_workload_identity_client()
if client is None:
raise RuntimeError("Workload identity client is not configured")
claims = populate_claims_for_workload(unified_job)
kwargs = {"claims": claims, "scope": scope, "audience": audience}
if workload_ttl_seconds:
kwargs["workload_ttl_seconds"] = workload_ttl_seconds
return client.request_workload_jwt(**kwargs).jwt
def with_path_cleanup(f):
@functools.wraps(f)
def _wrapped(self, *args, **kwargs):
@@ -204,7 +189,6 @@ def dispatch_waiting_jobs(binder):
if not kwargs:
kwargs = {}
binder.control('run', data={'task': serialize_task(uj._get_task_class()), 'args': [uj.id], 'kwargs': kwargs, 'uuid': uj.celery_task_id})
UnifiedJob.objects.filter(pk=uj.pk, status='waiting').update(status='running', start_args='')
class BaseTask(object):
@@ -219,60 +203,6 @@ class BaseTask(object):
self.update_attempts = int(getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE) / 5)
self.runner_callback = self.callback_class(model=self.model)
@functools.cached_property
def _credentials(self):
"""
Credentials for the task execution.
Fetches credentials once using build_credentials_list() and stores
them for the duration of the task to avoid redundant database queries.
"""
credentials_list = self.build_credentials_list(self.instance)
# Convert to list to prevent re-evaluation of QuerySet
return list(credentials_list)
def populate_workload_identity_tokens(self):
"""
Populate credentials with workload identity tokens.
Sets the context on Credential objects that have input sources
using compatible external credential types.
"""
credential_input_sources = (
(credential.context, src)
for credential in self._credentials
for src in credential.input_sources.all()
if any(
field.get('id') == 'workload_identity_token' and field.get('internal')
for field in src.source_credential.credential_type.inputs.get('fields', [])
)
)
for credential_ctx, input_src in credential_input_sources:
if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
effective_timeout = self.get_instance_timeout(self.instance)
workload_ttl = effective_timeout if effective_timeout else None
try:
jwt = retrieve_workload_identity_jwt(
self.instance,
audience=input_src.source_credential.get_input('jwt_aud'),
scope=AutomationControllerJobScope.name,
workload_ttl_seconds=workload_ttl,
)
# Store token keyed by input source PK, since a credential can have
# multiple input sources (one per field), each potentially with a different audience
credential_ctx[input_src.pk] = {"workload_identity_token": jwt}
except Exception as e:
self.instance.job_explanation = (
f'Could not generate workload identity token for credential {input_src.source_credential.name} used in this job. Error:\n{e}'
)
self.instance.status = 'error'
self.instance.save()
else:
self.instance.job_explanation = (
f'Flag FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED is not enabled, required for credential {input_src.source_credential.name} used in this job.'
)
self.instance.status = 'error'
self.instance.save()
def update_model(self, pk, _attempt=0, **updates):
return update_model(self.model, pk, _attempt=0, _max_attempts=self.update_attempts, **updates)
@@ -424,19 +354,6 @@ class BaseTask(object):
private_data_files['credentials'][credential] = self.write_private_data_file(private_data_dir, None, data, sub_dir='env')
for credential, data in private_data.get('certificates', {}).items():
self.write_private_data_file(private_data_dir, 'ssh_key_data-cert.pub', data, sub_dir=os.path.join('artifacts', str(self.instance.id)))
# Copy vendor collections to private_data_dir for indirect node counting
# This makes external query files available to the callback plugin in EEs
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
vendor_src = '/var/lib/awx/vendor_collections'
vendor_dest = os.path.join(private_data_dir, 'vendor_collections')
if os.path.exists(vendor_src):
try:
shutil.copytree(vendor_src, vendor_dest)
logger.debug(f"Copied vendor collections from {vendor_src} to {vendor_dest}")
except Exception as e:
logger.warning(f"Failed to copy vendor collections: {e}")
return private_data_files, ssh_key_data
def build_passwords(self, instance, runtime_passwords):
@@ -510,7 +427,6 @@ class BaseTask(object):
return []
def get_instance_timeout(self, instance):
"""Return the effective job timeout in seconds."""
global_timeout_setting_name = instance._global_timeout_setting()
if global_timeout_setting_name:
global_timeout = getattr(settings, global_timeout_setting_name, 0)
@@ -619,32 +535,48 @@ class BaseTask(object):
def should_use_fact_cache(self):
return False
def transition_status(self, pk: int) -> bool:
"""Atomically transition status to running, if False returned, another process got it"""
with transaction.atomic():
# Explanation of parts for the fetch:
# .values - avoid loading a full object, this is known to lead to deadlocks due to signals
# the signals load other related rows which another process may be locking, and happens in practice
# of=('self',) - keeps FK tables out of the lock list, another way deadlocks can happen
# .get - just load the single job
instance_data = UnifiedJob.objects.select_for_update(of=('self',)).values('status', 'cancel_flag').get(pk=pk)
# If status is not waiting (obtained under lock) then this process does not have clearence to run
if instance_data['status'] == 'waiting':
if instance_data['cancel_flag']:
updated_status = 'canceled'
else:
updated_status = 'running'
# Explanation of the update:
# .filter - again, do not load the full object
# .update - a bulk update on just that one row, avoid loading unintended data
UnifiedJob.objects.filter(pk=pk).update(status=updated_status, start_args='')
elif instance_data['status'] == 'running':
logger.info(f'Job {pk} is being ran by another process, exiting')
return False
return True
@with_path_cleanup
@with_signal_handling
def run(self, pk, **kwargs):
"""
Run the job/task and capture its output.
"""
if not self.instance: # Used to skip fetch for local runs
# Load the instance
self.instance = self.update_model(pk)
# status should be "running" from dispatch_waiting_jobs,
# but may still be "waiting" if the worker picked this up before the status update landed.
if self.instance.status == 'waiting':
UnifiedJob.objects.filter(pk=pk).update(status="running", start_args='')
self.instance.refresh_from_db()
if not self.transition_status(pk):
logger.info(f'Job {pk} is being ran by another process, exiting')
return
# Load the instance
self.instance = self.update_model(pk)
if self.instance.status != 'running':
logger.error(f'Not starting {self.instance.status} task pk={pk} because its status "{self.instance.status}" is not expected')
return
if self.instance.cancel_flag:
self.instance = self.update_model(pk, status='canceled')
self.instance.websocket_emit_status('canceled')
return
self.instance.websocket_emit_status("running")
status, rc = 'error', None
self.runner_callback.event_ct = 0
@@ -683,12 +615,6 @@ class BaseTask(object):
if not os.path.exists(settings.AWX_ISOLATION_BASE_PATH):
raise RuntimeError('AWX_ISOLATION_BASE_PATH=%s does not exist' % settings.AWX_ISOLATION_BASE_PATH)
if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
logger.info(f'Generating workload identity tokens for {self.instance.log_format}')
self.populate_workload_identity_tokens()
if self.instance.status == 'error':
raise RuntimeError('not starting %s task' % self.instance.status)
# May have to serialize the value
private_data_files, ssh_key_data = self.build_private_data_files(self.instance, private_data_dir)
passwords = self.build_passwords(self.instance, kwargs)
@@ -706,7 +632,7 @@ class BaseTask(object):
self.runner_callback.job_created = str(self.instance.created)
credentials = self._credentials
credentials = self.build_credentials_list(self.instance)
container_root = None
if settings.IS_K8S and isinstance(self.instance, ProjectUpdate):
@@ -1001,29 +927,6 @@ class RunJob(SourceControlMixin, BaseTask):
model = Job
event_model = JobEvent
def _extract_credentials_of_kind(self, kind: str):
return (cred for cred in self._credentials if cred.credential_type.kind == kind)
@property
def _machine_credential(self) -> object:
"""Get machine credential."""
return next(self._extract_credentials_of_kind('ssh'), None)
@property
def _vault_credentials(self) -> list[object]:
"""Get vault credentials."""
return list(self._extract_credentials_of_kind('vault'))
@property
def _network_credentials(self) -> list[object]:
"""Get network credentials."""
return list(self._extract_credentials_of_kind('net'))
@property
def _cloud_credentials(self) -> list[object]:
"""Get cloud credentials."""
return list(self._extract_credentials_of_kind('cloud'))
def build_private_data(self, job, private_data_dir):
"""
Returns a dict of the form
@@ -1041,7 +944,7 @@ class RunJob(SourceControlMixin, BaseTask):
}
"""
private_data = {'credentials': {}}
for credential in self._credentials:
for credential in job.credentials.prefetch_related('input_sources__source_credential').all():
# If we were sent SSH credentials, decrypt them and send them
# back (they will be written to a temporary file).
if credential.has_input('ssh_key_data'):
@@ -1057,14 +960,14 @@ class RunJob(SourceControlMixin, BaseTask):
and ansible-vault.
"""
passwords = super(RunJob, self).build_passwords(job, runtime_passwords)
cred = self._machine_credential
cred = job.machine_credential
if cred:
for field in ('ssh_key_unlock', 'ssh_password', 'become_password', 'vault_password'):
value = runtime_passwords.get(field, cred.get_input('password' if field == 'ssh_password' else field, default=''))
if value not in ('', 'ASK'):
passwords[field] = value
for cred in self._vault_credentials:
for cred in job.vault_credentials:
field = 'vault_password'
vault_id = cred.get_input('vault_id', default=None)
if vault_id:
@@ -1080,7 +983,7 @@ class RunJob(SourceControlMixin, BaseTask):
key unlock over network key unlock.
'''
if 'ssh_key_unlock' not in passwords:
for cred in self._network_credentials:
for cred in job.network_credentials:
if cred.inputs.get('ssh_key_unlock'):
passwords['ssh_key_unlock'] = runtime_passwords.get('ssh_key_unlock', cred.get_input('ssh_key_unlock', default=''))
break
@@ -1115,11 +1018,11 @@ class RunJob(SourceControlMixin, BaseTask):
# Set environment variables for cloud credentials.
cred_files = private_data_files.get('credentials', {})
for cloud_cred in self._cloud_credentials:
for cloud_cred in job.cloud_credentials:
if cloud_cred and cloud_cred.credential_type.namespace == 'openstack' and cred_files.get(cloud_cred, ''):
env['OS_CLIENT_CONFIG_FILE'] = get_incontainer_path(cred_files.get(cloud_cred, ''), private_data_dir)
for network_cred in self._network_credentials:
for network_cred in job.network_credentials:
env['ANSIBLE_NET_USERNAME'] = network_cred.get_input('username', default='')
env['ANSIBLE_NET_PASSWORD'] = network_cred.get_input('password', default='')
@@ -1162,11 +1065,6 @@ class RunJob(SourceControlMixin, BaseTask):
if 'callbacks_enabled' in config_values:
env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled']
# Add vendor collections path for external query file discovery
vendor_collections_path = os.path.join(CONTAINER_ROOT, 'vendor_collections')
env['ANSIBLE_COLLECTIONS_PATH'] = f"{vendor_collections_path}:{env['ANSIBLE_COLLECTIONS_PATH']}"
logger.debug(f"ANSIBLE_COLLECTIONS_PATH updated for vendor collections: {env['ANSIBLE_COLLECTIONS_PATH']}")
return env
def build_args(self, job, private_data_dir, passwords):
@@ -1174,7 +1072,7 @@ class RunJob(SourceControlMixin, BaseTask):
Build command line argument list for running ansible-playbook,
optionally using ssh-agent for public/private key authentication.
"""
creds = self._machine_credential
creds = job.machine_credential
ssh_username, become_username, become_method = '', '', ''
if creds:
@@ -1326,16 +1224,10 @@ class RunJob(SourceControlMixin, BaseTask):
return
if self.should_use_fact_cache() and self.runner_callback.artifacts_processed:
job.log_lifecycle("finish_job_fact_cache")
if job.inventory.kind == 'constructed':
hosts_qs = job.get_source_hosts_for_constructed_inventory()
else:
hosts_qs = job.inventory.hosts
finish_fact_cache(
hosts_qs,
artifacts_dir=os.path.join(private_data_dir, 'artifacts', str(job.id)),
job_id=job.id,
inventory_id=job.inventory_id,
job_created=job.created,
)
def final_run_hook(self, job, status, private_data_dir):

View File

@@ -393,9 +393,9 @@ def evaluate_policy(instance):
raise PolicyEvaluationError(_('Following certificate settings are missing for OPA_AUTH_TYPE=Certificate: {}').format(cert_settings_missing))
query_paths = [
('Organization', instance.organization.opa_query_path if instance.organization else None),
('Inventory', instance.inventory.opa_query_path if instance.inventory else None),
('Job template', instance.job_template.opa_query_path if instance.job_template else None),
('Organization', instance.organization.opa_query_path),
('Inventory', instance.inventory.opa_query_path),
('Job template', instance.job_template.opa_query_path),
]
violations = dict()
errors = dict()

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 1.0.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection v1.5.0. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 1.5.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection v3.0.0. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 3.0.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,21 +0,0 @@
---
# Generated by Claude Opus 4.6 (claude-opus-4-6).
- hosts: all
vars:
extra_value: ""
gather_facts: false
connection: local
tasks:
- name: set a custom fact
set_fact:
foo: "bar{{ extra_value }}"
bar:
a:
b:
- "c"
- "d"
cacheable: true
- name: sleep to create overlap window for concurrent job testing
wait_for:
timeout: 2

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v1_0_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v1_5_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v3_0_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -1,7 +1,5 @@
import pytest
from ansible_base.lib.testing.util import feature_flag_enabled, feature_flag_disabled
from awx.main.models import CredentialInputSource
from awx.api.versioning import reverse
@@ -318,60 +316,3 @@ def test_create_credential_input_source_with_already_used_input_returns_400(post
]
all_responses = [post(list_url, params, admin) for params in all_params]
assert all_responses.pop().status_code == 400
@pytest.mark.django_db
def test_credential_input_source_passes_workload_identity_token_when_flag_enabled(vault_credential, external_credential, mocker):
"""Test that workload_identity_token is passed to backend when flag is enabled."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
# Add workload_identity_token as an internal field on the external credential type
# so get_input_value resolves it from the per-input-source context
external_credential.credential_type.inputs['fields'].append(
{'id': 'workload_identity_token', 'label': 'Workload Identity Token', 'type': 'string', 'internal': True}
)
# Create an input source
input_source = CredentialInputSource.objects.create(
target_credential=vault_credential,
source_credential=external_credential,
input_field_name='vault_password',
metadata={'key': 'test_key'},
)
# Mock the credential plugin backend
mock_backend = mocker.patch.object(external_credential.credential_type.plugin, 'backend', autospec=True, return_value='test_value')
# Call with context keyed by input source PK
test_context = {input_source.pk: {'workload_identity_token': 'jwt_token_here'}}
result = input_source.get_input_value(context=test_context)
# Verify backend was called with workload_identity_token
assert result == 'test_value'
call_kwargs = mock_backend.call_args[1]
assert call_kwargs['workload_identity_token'] == 'jwt_token_here'
assert call_kwargs['key'] == 'test_key'
@pytest.mark.django_db
def test_credential_input_source_skips_workload_identity_token_when_flag_disabled(vault_credential, external_credential, mocker):
"""Test that workload_identity_token is NOT passed when flag is disabled."""
with feature_flag_disabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
# Create an input source
input_source = CredentialInputSource.objects.create(
target_credential=vault_credential,
source_credential=external_credential,
input_field_name='vault_password',
metadata={'key': 'test_key'},
)
# Mock the credential plugin backend
mock_backend = mocker.patch.object(external_credential.credential_type.plugin, 'backend', autospec=True, return_value='test_value')
# Call with context containing workload_identity_token but NO internal field defined,
# simulating a flag-disabled scenario where tokens are not generated upstream
test_context = {input_source.pk: {'workload_identity_token': 'jwt_token_here'}}
result = input_source.get_input_value(context=test_context)
# Verify backend was called WITHOUT workload_identity_token since the credential type
# does not define it as an internal field (flag-disabled path doesn't register it)
assert result == 'test_value'
call_kwargs = mock_backend.call_args[1]
assert 'workload_identity_token' not in call_kwargs
assert call_kwargs['key'] == 'test_key'

View File

@@ -485,3 +485,47 @@ class TestJobTemplateCallbackProxyIntegration:
expect=400,
**headers
)
@override_settings(REMOTE_HOST_HEADERS=['HTTP_X_FROM_THE_LOAD_BALANCER', 'REMOTE_ADDR', 'REMOTE_HOST'], PROXY_IP_ALLOWED_LIST=[])
def test_only_first_entry_in_comma_separated_header_is_considered(self, job_template, admin_user, post):
"""
Test that only the first entry in a comma-separated header value is used for host matching.
This is important for X-Forwarded-For style headers where the format is "client, proxy1, proxy2".
Only the original client (first entry) should be matched against inventory hosts.
"""
# Create host that matches the SECOND entry in the comma-separated list
job_template.inventory.hosts.create(name='second-host.example.com')
headers = {
# First entry is 'first-host.example.com', second is 'second-host.example.com'
# Only the first should be considered, so this should NOT match
'HTTP_X_FROM_THE_LOAD_BALANCER': 'first-host.example.com, second-host.example.com',
'REMOTE_ADDR': 'unrelated-addr',
'REMOTE_HOST': 'unrelated-host',
}
# Should return 400 because only 'first-host.example.com' is considered,
# and that host is NOT in the inventory
r = post(
url=reverse('api:job_template_callback', kwargs={'pk': job_template.pk}), data={'host_config_key': 'abcd'}, user=admin_user, expect=400, **headers
)
assert r.data['msg'] == 'No matching host could be found!'
@override_settings(REMOTE_HOST_HEADERS=['HTTP_X_FROM_THE_LOAD_BALANCER', 'REMOTE_ADDR', 'REMOTE_HOST'], PROXY_IP_ALLOWED_LIST=[])
def test_first_entry_in_comma_separated_header_matches(self, job_template, admin_user, post):
"""
Test that the first entry in a comma-separated header value correctly matches an inventory host.
"""
# Create host that matches the FIRST entry in the comma-separated list
job_template.inventory.hosts.create(name='first-host.example.com')
headers = {
# First entry is 'first-host.example.com', second is 'second-host.example.com'
# The first entry matches the inventory host
'HTTP_X_FROM_THE_LOAD_BALANCER': 'first-host.example.com, second-host.example.com',
'REMOTE_ADDR': 'unrelated-addr',
'REMOTE_HOST': 'unrelated-host',
}
# Should return 201 because 'first-host.example.com' is the first entry and matches
post(url=reverse('api:job_template_callback', kwargs={'pk': job_template.pk}), data={'host_config_key': 'abcd'}, user=admin_user, expect=201, **headers)

View File

@@ -1,163 +0,0 @@
"""
Tests for OIDC workload identity credential type feature flag.
The FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED flag is an install-time flag that
controls whether OIDC credential types are loaded into the registry at startup.
When disabled, OIDC credential types are not loaded and do not exist in the database.
"""
import pytest
from unittest import mock
from django.test import override_settings
from awx.main.constants import OIDC_CREDENTIAL_TYPE_NAMESPACES
from awx.main.models.credential import CredentialType, ManagedCredentialType, load_credentials
from awx.api.versioning import reverse
@pytest.fixture
def reload_credentials_with_flag(django_db_setup, django_db_blocker):
"""
Fixture that reloads credentials with a specific flag state.
This simulates what happens at application startup.
"""
# Save original registry state
original_registry = ManagedCredentialType.registry.copy()
def _reload(flag_enabled):
with django_db_blocker.unblock():
# Clear the entire registry before reloading
ManagedCredentialType.registry.clear()
# Reload credentials with the specified flag state
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=flag_enabled):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
load_credentials()
# Sync to database
CredentialType.setup_tower_managed_defaults(lock=False)
# In tests, the session fixture pre-loads all credential types into the DB.
# Remove OIDC types when testing the disabled state so the API test is accurate.
if not flag_enabled:
CredentialType.objects.filter(namespace__in=OIDC_CREDENTIAL_TYPE_NAMESPACES).delete()
yield _reload
# Restore original registry state after tests
ManagedCredentialType.registry.clear()
ManagedCredentialType.registry.update(original_registry)
@pytest.fixture
def isolated_registry():
"""Save and restore the ManagedCredentialType registry, with full isolation via mocked entry_points."""
original_registry = ManagedCredentialType.registry.copy()
ManagedCredentialType.registry.clear()
yield
ManagedCredentialType.registry.clear()
ManagedCredentialType.registry.update(original_registry)
def _make_mock_entry_point(name):
"""Create a mock entry point that mimics a credential plugin."""
ep = mock.MagicMock()
ep.name = name
ep.value = f'test_plugin:{name}'
plugin = mock.MagicMock(spec=[])
ep.load.return_value = plugin
return ep
def _mock_entry_points_factory(managed_names, supported_names):
"""Return a side_effect function for mocking entry_points() with controlled plugins."""
managed = [_make_mock_entry_point(n) for n in managed_names]
supported = [_make_mock_entry_point(n) for n in supported_names]
def _entry_points(group):
if group == 'awx_plugins.managed_credentials':
return managed
elif group == 'awx_plugins.managed_credentials.supported':
return supported
return []
return _entry_points
# --- Unit tests for load_credentials() registry behavior ---
def test_oidc_types_in_registry_when_flag_enabled(isolated_registry):
"""Test that OIDC credential types are added to the registry when flag is enabled."""
mock_eps = _mock_entry_points_factory(
managed_names=['ssh', 'vault'],
supported_names=['hashivault-kv-oidc', 'hashivault-ssh-oidc'],
)
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=True):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
with mock.patch('awx.main.models.credential.entry_points', side_effect=mock_eps):
load_credentials()
for ns in OIDC_CREDENTIAL_TYPE_NAMESPACES:
assert ns in ManagedCredentialType.registry, f"{ns} should be in registry when flag is enabled"
assert 'ssh' in ManagedCredentialType.registry
assert 'vault' in ManagedCredentialType.registry
def test_oidc_types_not_in_registry_when_flag_disabled(isolated_registry):
"""Test that OIDC credential types are excluded from the registry when flag is disabled."""
mock_eps = _mock_entry_points_factory(
managed_names=['ssh', 'vault'],
supported_names=['hashivault-kv-oidc', 'hashivault-ssh-oidc'],
)
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=False):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
with mock.patch('awx.main.models.credential.entry_points', side_effect=mock_eps):
load_credentials()
for ns in OIDC_CREDENTIAL_TYPE_NAMESPACES:
assert ns not in ManagedCredentialType.registry, f"{ns} should not be in registry when flag is disabled"
# Non-OIDC types should still be loaded
assert 'ssh' in ManagedCredentialType.registry
assert 'vault' in ManagedCredentialType.registry
def test_oidc_namespaces_constant():
"""Test that OIDC_CREDENTIAL_TYPE_NAMESPACES contains the expected namespaces."""
assert 'hashivault-kv-oidc' in OIDC_CREDENTIAL_TYPE_NAMESPACES
assert 'hashivault-ssh-oidc' in OIDC_CREDENTIAL_TYPE_NAMESPACES
assert len(OIDC_CREDENTIAL_TYPE_NAMESPACES) == 2
# --- Functional API tests ---
@pytest.mark.django_db
def test_oidc_types_loaded_when_flag_enabled(get, admin, reload_credentials_with_flag):
"""Test that OIDC credential types are visible in the API when flag is enabled."""
reload_credentials_with_flag(flag_enabled=True)
response = get(reverse('api:credential_type_list'), admin)
assert response.status_code == 200
namespaces = [ct['namespace'] for ct in response.data['results']]
assert 'hashivault-kv-oidc' in namespaces
assert 'hashivault-ssh-oidc' in namespaces
@pytest.mark.django_db
def test_oidc_types_not_loaded_when_flag_disabled(get, admin, reload_credentials_with_flag):
"""Test that OIDC credential types are not visible in the API when flag is disabled."""
reload_credentials_with_flag(flag_enabled=False)
response = get(reverse('api:credential_type_list'), admin)
assert response.status_code == 200
namespaces = [ct['namespace'] for ct in response.data['results']]
assert 'hashivault-kv-oidc' not in namespaces
assert 'hashivault-ssh-oidc' not in namespaces
# Verify they're also not in the database
assert not CredentialType.objects.filter(namespace='hashivault-kv-oidc').exists()
assert not CredentialType.objects.filter(namespace='hashivault-ssh-oidc').exists()

View File

@@ -1,3 +1,4 @@
from datetime import date
from unittest import mock
import pytest
@@ -252,7 +253,7 @@ def test_user_verify_attribute_created(admin, get):
resp = get(reverse('api:user_detail', kwargs={'pk': admin.pk}), admin)
assert resp.data['created'] == admin.date_joined
past = "2020-01-01T00:00:00Z"
past = date(2020, 1, 1).isoformat()
for op, count in (('gt', 1), ('lt', 0)):
resp = get(reverse('api:user_list') + f'?created__{op}={past}', admin)
assert resp.data['count'] == count

View File

@@ -48,7 +48,7 @@ class TestCallbackBrokerWorker(TransactionTestCase):
worker = CallbackBrokerWorker()
events = [InventoryUpdateEvent(uuid=str(uuid4()), **self.event_create_kwargs())]
worker.buff = {InventoryUpdateEvent: events}
worker.flush(force=True)
worker.flush()
assert worker.buff.get(InventoryUpdateEvent, []) == []
assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 1
@@ -61,7 +61,7 @@ class TestCallbackBrokerWorker(TransactionTestCase):
InventoryUpdateEvent(uuid=str(uuid4()), stdout='good2', **kwargs),
]
worker.buff = {InventoryUpdateEvent: events.copy()}
worker.flush(force=True)
worker.flush()
assert InventoryUpdateEvent.objects.filter(uuid=events[0].uuid).count() == 1
assert InventoryUpdateEvent.objects.filter(uuid=events[1].uuid).count() == 0
assert InventoryUpdateEvent.objects.filter(uuid=events[2].uuid).count() == 1
@@ -71,7 +71,7 @@ class TestCallbackBrokerWorker(TransactionTestCase):
worker = CallbackBrokerWorker()
events = [InventoryUpdateEvent(uuid=str(uuid4()), **self.event_create_kwargs())]
worker.buff = {InventoryUpdateEvent: events.copy()}
worker.flush(force=True)
worker.flush()
# put current saved event in buffer (error case)
worker.buff = {InventoryUpdateEvent: [InventoryUpdateEvent.objects.get(uuid=events[0].uuid)]}
@@ -113,7 +113,7 @@ class TestCallbackBrokerWorker(TransactionTestCase):
with mock.patch.object(InventoryUpdateEvent.objects, 'bulk_create', side_effect=ValueError):
with mock.patch.object(events[0], 'save', side_effect=ValueError):
worker.flush(force=True)
worker.flush()
assert "\x00" not in events[0].stdout

View File

@@ -10,23 +10,9 @@ from django.test.utils import override_settings
@pytest.mark.django_db
def test_multiple_hybrid_instances():
for i in range(3):
Instance.objects.create(hostname=f'foo{i}', node_type='hybrid')
assert is_ha_environment()
@pytest.mark.django_db
def test_double_control_instances():
def test_multiple_instances():
for i in range(2):
Instance.objects.create(hostname=f'foo{i}', node_type='control')
assert is_ha_environment()
@pytest.mark.django_db
def test_mix_hybrid_control_instances():
Instance.objects.create(hostname='control_node', node_type='control')
Instance.objects.create(hostname='hybrid_node', node_type='hybrid')
Instance.objects.create(hostname=f'foo{i}', node_type='hybrid')
assert is_ha_environment()

View File

@@ -1,6 +1,6 @@
import pytest
from awx.main.models import JobTemplate, Job, JobHostSummary, WorkflowJob, Inventory, Host, Project, Organization
from awx.main.models import JobTemplate, Job, JobHostSummary, WorkflowJob, Inventory, Project, Organization
@pytest.mark.django_db
@@ -87,47 +87,3 @@ class TestSlicingModels:
unified_job = job_template.create_unified_job(job_slice_count=2)
assert isinstance(unified_job, Job)
@pytest.mark.django_db
class TestGetSourceHostsForConstructedInventory:
"""Tests for Job.get_source_hosts_for_constructed_inventory"""
def test_returns_source_hosts_via_instance_id(self):
"""Constructed hosts with instance_id pointing to source hosts are resolved correctly."""
org = Organization.objects.create(name='test-org')
inv_input = Inventory.objects.create(organization=org, name='input-inv')
source_host1 = inv_input.hosts.create(name='host1')
source_host2 = inv_input.hosts.create(name='host2')
inv_constructed = Inventory.objects.create(organization=org, name='constructed-inv', kind='constructed')
inv_constructed.input_inventories.add(inv_input)
Host.objects.create(inventory=inv_constructed, name='host1', instance_id=str(source_host1.id))
Host.objects.create(inventory=inv_constructed, name='host2', instance_id=str(source_host2.id))
job = Job.objects.create(name='test-job', inventory=inv_constructed)
result = job.get_source_hosts_for_constructed_inventory()
assert set(result.values_list('id', flat=True)) == {source_host1.id, source_host2.id}
def test_no_inventory_returns_empty(self):
"""A job with no inventory returns an empty queryset."""
job = Job.objects.create(name='test-job')
result = job.get_source_hosts_for_constructed_inventory()
assert result.count() == 0
def test_ignores_hosts_without_instance_id(self):
"""Hosts with empty instance_id are excluded from the result."""
org = Organization.objects.create(name='test-org')
inv_input = Inventory.objects.create(organization=org, name='input-inv')
source_host = inv_input.hosts.create(name='host1')
inv_constructed = Inventory.objects.create(organization=org, name='constructed-inv', kind='constructed')
inv_constructed.input_inventories.add(inv_input)
Host.objects.create(inventory=inv_constructed, name='host1', instance_id=str(source_host.id))
Host.objects.create(inventory=inv_constructed, name='host-no-ref', instance_id='')
job = Job.objects.create(name='test-job', inventory=inv_constructed)
result = job.get_source_hosts_for_constructed_inventory()
assert list(result.values_list('id', flat=True)) == [source_host.id]

View File

@@ -1,274 +0,0 @@
# Generated by Claude Opus 4.6 (claude-opus-4-6)
#
# Test file for cancel + dependency chain behavior and workflow cancel propagation.
#
# These tests verify:
#
# 1. TaskManager.process_job_dep_failures() correctly distinguishes canceled vs
# failed dependencies in the job_explanation message.
#
# 2. TaskManager.process_pending_tasks() transitions pending jobs with
# cancel_flag=True directly to canceled status.
#
# 3. WorkflowManager + TaskManager together cancel all spawned jobs in a
# workflow and finalize the workflow as canceled.
import pytest
from unittest import mock
from awx.main.scheduler import TaskManager, DependencyManager, WorkflowManager
from awx.main.models import JobTemplate, ProjectUpdate, WorkflowApproval, WorkflowJobTemplate
from awx.main.models.workflow import WorkflowApprovalTemplate
from . import create_job
@pytest.fixture
def scm_on_launch_objects(job_template_factory):
"""Create a job template with a project configured for scm_update_on_launch."""
objects = job_template_factory(
'jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred',
)
p = objects.project
p.scm_update_on_launch = True
p.scm_update_cache_timeout = 0
p.save(skip_update=True)
return objects
def _create_job_with_dependency(objects):
"""Create a pending job and run DependencyManager to produce its project update dependency.
Returns (job, project_update).
"""
j = create_job(objects.job_template, dependencies_processed=False)
with mock.patch('awx.main.models.unified_jobs.UnifiedJobTemplate.update'):
DependencyManager().schedule()
assert j.dependent_jobs.count() == 1
pu = j.dependent_jobs.first()
assert isinstance(pu.get_real_instance(), ProjectUpdate)
return j, pu
@pytest.mark.django_db
class TestCanceledDependencyFailsBlockedJob:
"""When a dependency project update is canceled or failed, the task manager
should fail the blocked job via process_job_dep_failures."""
def test_canceled_dependency_fails_blocked_job(self, controlplane_instance_group, scm_on_launch_objects):
"""A canceled dependency causes the blocked job to be failed with
a 'Previous Task Canceled' explanation."""
j, pu = _create_job_with_dependency(scm_on_launch_objects)
ProjectUpdate.objects.filter(pk=pu.pk).update(status='canceled', cancel_flag=True)
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
j.refresh_from_db()
assert j.status == 'failed'
assert 'Previous Task Canceled' in j.job_explanation
def test_failed_dependency_fails_blocked_job(self, controlplane_instance_group, scm_on_launch_objects):
"""A failed dependency causes the blocked job to be failed with
a 'Previous Task Failed' explanation."""
j, pu = _create_job_with_dependency(scm_on_launch_objects)
ProjectUpdate.objects.filter(pk=pu.pk).update(status='failed')
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
j.refresh_from_db()
assert j.status == 'failed'
assert 'Previous Task Failed' in j.job_explanation
@pytest.mark.django_db
class TestTaskManagerCancelsPendingJobsWithCancelFlag:
"""When the task manager encounters pending jobs that have cancel_flag set,
it should transition them directly to canceled status."""
def test_pending_job_with_cancel_flag_is_canceled(self, controlplane_instance_group, job_template_factory):
"""A pending job with cancel_flag=True is transitioned to canceled
by the task manager without being started."""
objects = job_template_factory(
'jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred',
)
j = create_job(objects.job_template)
j.cancel_flag = True
j.save(update_fields=['cancel_flag'])
with mock.patch("awx.main.scheduler.TaskManager.start_task") as mock_start:
TaskManager().schedule()
j.refresh_from_db()
assert j.status == 'canceled'
assert 'canceled before it started' in j.job_explanation
assert not mock_start.called
def test_pending_job_without_cancel_flag_is_not_canceled(self, controlplane_instance_group, job_template_factory):
"""A normal pending job without cancel_flag should not be canceled
by the task manager (sanity check)."""
objects = job_template_factory(
'jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred',
)
j = create_job(objects.job_template)
with mock.patch("awx.main.scheduler.TaskManager.start_task"):
TaskManager().schedule()
j.refresh_from_db()
assert j.status != 'canceled'
def test_multiple_pending_jobs_with_cancel_flag_bulk_canceled(self, controlplane_instance_group, job_template_factory):
"""Multiple pending jobs with cancel_flag=True are all transitioned
to canceled in a single task manager cycle."""
objects = job_template_factory(
'jt',
organization='org1',
project='proj',
inventory='inv',
credential='cred',
)
jt = objects.job_template
jt.allow_simultaneous = True
jt.save()
jobs = []
for _ in range(3):
j = create_job(jt)
j.cancel_flag = True
j.save(update_fields=['cancel_flag'])
jobs.append(j)
with mock.patch("awx.main.scheduler.TaskManager.start_task") as mock_start:
TaskManager().schedule()
for j in jobs:
j.refresh_from_db()
assert j.status == 'canceled', f"Job {j.id} should be canceled but is {j.status}"
assert 'canceled before it started' in j.job_explanation
assert not mock_start.called
@pytest.mark.django_db
class TestWorkflowCancelFinalizesWorkflow:
"""When a workflow job is canceled, the WorkflowManager cancels spawned child
jobs (setting cancel_flag), the TaskManager transitions those pending jobs to
canceled, and a final WorkflowManager pass finalizes the workflow as canceled."""
def test_cancel_workflow_with_parallel_nodes(self, inventory, project, controlplane_instance_group):
"""Create a workflow with parallel nodes, cancel it after one job is
running, and verify all jobs and the workflow reach canceled status."""
jt = JobTemplate.objects.create(allow_simultaneous=False, inventory=inventory, project=project, playbook='helloworld.yml')
wfjt = WorkflowJobTemplate.objects.create(name='test-cancel-wf')
for _ in range(4):
wfjt.workflow_nodes.create(unified_job_template=jt)
wj = wfjt.create_unified_job()
wj.signal_start()
# TaskManager transitions workflow job to running via start_task
TaskManager().schedule()
wj.refresh_from_db()
assert wj.status == 'running'
# WorkflowManager spawns jobs for all 4 nodes
WorkflowManager().schedule()
assert jt.jobs.count() == 4
# Simulate one job running (blocking the others via allow_simultaneous=False)
first_job = jt.jobs.order_by('created').first()
first_job.status = 'running'
first_job.celery_task_id = 'fake-task-id'
first_job.controller_node = 'test-node'
first_job.save(update_fields=['status', 'celery_task_id', 'controller_node'])
# Cancel the workflow
wj.cancel_flag = True
wj.save(update_fields=['cancel_flag'])
# WorkflowManager sees cancel_flag, calls cancel_node_jobs() which sets
# cancel_flag on all child jobs
with mock.patch('awx.main.models.unified_jobs.UnifiedJob.cancel_dispatcher_process'):
WorkflowManager().schedule()
# The running job won't actually stop in tests (no dispatcher), simulate it
first_job.status = 'canceled'
first_job.save(update_fields=['status'])
# TaskManager processes remaining pending jobs with cancel_flag set
with mock.patch("awx.main.scheduler.TaskManager.start_task") as mock_start:
DependencyManager().schedule()
TaskManager().schedule()
for job in jt.jobs.all():
job.refresh_from_db()
assert job.status == 'canceled', f"Job {job.id} should be canceled but is {job.status}"
assert not mock_start.called
# Final WorkflowManager pass finalizes the workflow
WorkflowManager().schedule()
wj.refresh_from_db()
assert wj.status == 'canceled'
def test_cancel_workflow_with_approval_node(self, controlplane_instance_group):
"""Create a workflow with a pending approval node and a downstream job
node. Cancel the workflow and verify the approval is directly canceled
by the WorkflowManager (since approvals are excluded from TaskManager),
the downstream node is marked do_not_run, and the workflow finalizes."""
approval_template = WorkflowApprovalTemplate.objects.create(name='test-approval', timeout=0)
wfjt = WorkflowJobTemplate.objects.create(name='test-cancel-approval-wf')
approval_node = wfjt.workflow_nodes.create(unified_job_template=approval_template)
# Add a downstream node (just another approval to keep it simple)
downstream_template = WorkflowApprovalTemplate.objects.create(name='test-downstream', timeout=0)
downstream_node = wfjt.workflow_nodes.create(unified_job_template=downstream_template)
approval_node.success_nodes.add(downstream_node)
wj = wfjt.create_unified_job()
wj.signal_start()
# TaskManager transitions workflow to running
TaskManager().schedule()
wj.refresh_from_db()
assert wj.status == 'running'
# WorkflowManager spawns the approval (root node only, downstream waits)
WorkflowManager().schedule()
assert WorkflowApproval.objects.filter(unified_job_node__workflow_job=wj).count() == 1
approval_job = WorkflowApproval.objects.get(unified_job_node__workflow_job=wj)
assert approval_job.status == 'pending'
# Cancel the workflow
wj.cancel_flag = True
wj.save(update_fields=['cancel_flag'])
# WorkflowManager should cancel the approval directly and mark
# the downstream node as do_not_run
WorkflowManager().schedule()
approval_job.refresh_from_db()
assert approval_job.status == 'canceled', f"Approval should be canceled directly by WorkflowManager but is {approval_job.status}"
# Downstream node should be marked do_not_run with no job spawned
downstream_wj_node = wj.workflow_nodes.get(unified_job_template=downstream_template)
assert downstream_wj_node.do_not_run is True
assert downstream_wj_node.job is None
# Workflow should finalize as canceled in the same pass
wj.refresh_from_db()
assert wj.status == 'canceled'

View File

@@ -1,223 +0,0 @@
"""Functional tests for start_fact_cache / finish_fact_cache.
These tests use real database objects (via pytest-django) and real files
on disk, but do not launch jobs or subprocesses. Fact files are written
by start_fact_cache and then manipulated to simulate ansible output
before calling finish_fact_cache.
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
import json
import os
import time
from datetime import timedelta
import pytest
from django.utils.timezone import now
from awx.main.models import Host, Inventory
from awx.main.tasks.facts import start_fact_cache, finish_fact_cache
@pytest.fixture
def artifacts_dir(tmp_path):
d = tmp_path / 'artifacts'
d.mkdir()
return str(d)
@pytest.mark.django_db
class TestFinishFactCacheScoping:
"""finish_fact_cache must only update hosts matched by the provided queryset."""
def test_same_hostname_different_inventories(self, organization, artifacts_dir):
"""Two inventories share a hostname; only the targeted one should be updated.
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
inv1 = Inventory.objects.create(organization=organization, name='scope-inv1')
inv2 = Inventory.objects.create(organization=organization, name='scope-inv2')
host1 = inv1.hosts.create(name='shared')
host2 = inv2.hosts.create(name='shared')
# Give both hosts initial facts
for h in (host1, host2):
h.ansible_facts = {'original': True}
h.ansible_facts_modified = now()
h.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
# start_fact_cache writes reference files for inv1's hosts
start_fact_cache(inv1.hosts.all(), artifacts_dir=artifacts_dir, timeout=0, inventory_id=inv1.id)
# Simulate ansible writing new facts for 'shared'
fact_file = os.path.join(artifacts_dir, 'fact_cache', 'shared')
future = time.time() + 60
with open(fact_file, 'w') as f:
json.dump({'updated': True}, f)
os.utime(fact_file, (future, future))
# finish with inv1's hosts as the queryset
finish_fact_cache(inv1.hosts, artifacts_dir=artifacts_dir, inventory_id=inv1.id)
host1.refresh_from_db()
host2.refresh_from_db()
assert host1.ansible_facts == {'updated': True}
assert host2.ansible_facts == {'original': True}, 'Host in a different inventory was modified despite not being in the queryset'
@pytest.mark.django_db
class TestFinishFactCacheConcurrentProtection:
"""finish_fact_cache must not clear facts that a concurrent job updated."""
def test_no_clear_when_no_file_was_written(self, organization, artifacts_dir):
"""Host with no prior facts should not have facts cleared when file is missing.
Generated by Claude Opus 4.6 (claude-opus-4-6).
start_fact_cache records hosts_cached[host] = False for hosts with no
prior facts (no file written). finish_fact_cache should skip the clear
for these hosts because the missing file is expected, not a clear signal.
"""
inv = Inventory.objects.create(organization=organization, name='concurrent-inv')
host = inv.hosts.create(name='target')
job_created = now() - timedelta(minutes=5)
# start_fact_cache records host with False (no facts → no file written)
start_fact_cache(inv.hosts.all(), artifacts_dir=artifacts_dir, timeout=0, inventory_id=inv.id)
# Simulate a concurrent job updating this host's facts AFTER our job was created
host.ansible_facts = {'from_concurrent_job': True}
host.ansible_facts_modified = now()
host.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
# The fact file is missing because start_fact_cache never wrote one.
# finish_fact_cache should skip this host entirely.
finish_fact_cache(
inv.hosts,
artifacts_dir=artifacts_dir,
inventory_id=inv.id,
job_created=job_created,
)
host.refresh_from_db()
assert host.ansible_facts == {'from_concurrent_job': True}, 'Facts were cleared for a host that never had a fact file written'
def test_skip_clear_when_facts_modified_after_job_created(self, organization, artifacts_dir):
"""If a file was written and then deleted, but facts were concurrently updated, skip clear.
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
inv = Inventory.objects.create(organization=organization, name='concurrent-written-inv')
host = inv.hosts.create(name='target')
old_time = now() - timedelta(hours=1)
host.ansible_facts = {'original': True}
host.ansible_facts_modified = old_time
host.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
job_created = now() - timedelta(minutes=5)
# start_fact_cache writes a file (host has facts → True in map)
start_fact_cache(inv.hosts.all(), artifacts_dir=artifacts_dir, timeout=0, inventory_id=inv.id)
# Remove the fact file (ansible didn't target this host via --limit)
os.remove(os.path.join(artifacts_dir, 'fact_cache', host.name))
# Simulate a concurrent job updating this host's facts AFTER our job was created
host.ansible_facts = {'from_concurrent_job': True}
host.ansible_facts_modified = now()
host.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
finish_fact_cache(
inv.hosts,
artifacts_dir=artifacts_dir,
inventory_id=inv.id,
job_created=job_created,
)
host.refresh_from_db()
assert host.ansible_facts == {'from_concurrent_job': True}, 'Facts set by a concurrent job were cleared despite ansible_facts_modified > job_created'
def test_clear_when_facts_predate_job(self, organization, artifacts_dir):
"""If facts predate the job, a missing file should still clear them.
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
inv = Inventory.objects.create(organization=organization, name='clear-inv')
host = inv.hosts.create(name='stale')
old_time = now() - timedelta(hours=1)
host.ansible_facts = {'stale': True}
host.ansible_facts_modified = old_time
host.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
job_created = now() - timedelta(minutes=5)
start_fact_cache(inv.hosts.all(), artifacts_dir=artifacts_dir, timeout=0, inventory_id=inv.id)
# Remove the fact file to simulate ansible's clear_facts
os.remove(os.path.join(artifacts_dir, 'fact_cache', host.name))
finish_fact_cache(
inv.hosts,
artifacts_dir=artifacts_dir,
inventory_id=inv.id,
job_created=job_created,
)
host.refresh_from_db()
assert host.ansible_facts == {}, 'Stale facts should have been cleared when the fact file is missing ' 'and ansible_facts_modified predates job_created'
@pytest.mark.django_db
class TestConstructedInventoryFactCache:
"""finish_fact_cache with a constructed inventory queryset must target source hosts."""
def test_facts_resolve_to_source_host(self, organization, artifacts_dir):
"""Facts must be written to the source host, not the constructed copy.
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
from django.db.models.functions import Cast
inv_input = Inventory.objects.create(organization=organization, name='ci-input')
source_host = inv_input.hosts.create(name='webserver')
inv_constructed = Inventory.objects.create(organization=organization, name='ci-constructed', kind='constructed')
inv_constructed.input_inventories.add(inv_input)
constructed_host = Host.objects.create(
inventory=inv_constructed,
name='webserver',
instance_id=str(source_host.id),
)
# Build the same queryset that get_hosts_for_fact_cache uses
id_field = Host._meta.get_field('id')
source_qs = Host.objects.filter(id__in=inv_constructed.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field)))
# Give the source host initial facts so start_fact_cache writes a file
source_host.ansible_facts = {'role': 'web'}
source_host.ansible_facts_modified = now()
source_host.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
start_fact_cache(source_qs, artifacts_dir=artifacts_dir, timeout=0, inventory_id=inv_constructed.id)
# Simulate ansible writing updated facts
fact_file = os.path.join(artifacts_dir, 'fact_cache', 'webserver')
future = time.time() + 60
with open(fact_file, 'w') as f:
json.dump({'role': 'web', 'deployed': True}, f)
os.utime(fact_file, (future, future))
finish_fact_cache(source_qs, artifacts_dir=artifacts_dir, inventory_id=inv_constructed.id)
source_host.refresh_from_db()
constructed_host.refresh_from_db()
assert source_host.ansible_facts == {'role': 'web', 'deployed': True}
assert not constructed_host.ansible_facts, f'Facts were stored on the constructed host: {constructed_host.ansible_facts!r}'

View File

@@ -29,30 +29,3 @@ def test_cancel_flag_on_start(jt_linked, caplog):
job = Job.objects.get(id=job.id)
assert job.status == 'canceled'
@pytest.mark.django_db
def test_runjob_run_can_accept_waiting_status(jt_linked, mocker):
"""Test that RunJob.run() can accept a job in 'waiting' status and transition it to 'running'
before the pre_run_hook is called"""
job = jt_linked.create_unified_job()
job.status = 'waiting'
job.save()
status_at_pre_run = None
def capture_status(instance, private_data_dir):
nonlocal status_at_pre_run
instance.refresh_from_db()
status_at_pre_run = instance.status
mock_pre_run = mocker.patch.object(RunJob, 'pre_run_hook', side_effect=capture_status)
task = RunJob()
try:
task.run(job.id)
except Exception:
pass
mock_pre_run.assert_called_once()
assert status_at_pre_run == 'running'

View File

@@ -74,64 +74,47 @@ GLqbpJyX2r3p/Rmo6mLY71SqpA==
@pytest.mark.django_db
def test_default_cred_types():
expected = [
'aim',
'aws',
'aws_secretsmanager_credential',
'azure_kv',
'azure_rm',
'bitbucket_dc_token',
'centrify_vault_kv',
'conjur',
'controller',
'galaxy_api_token',
'gce',
'github_token',
'github_app_lookup',
'gitlab_token',
'gpg_public_key',
'hashivault_kv',
'hashivault_ssh',
'hcp_terraform',
'insights',
'kubernetes_bearer_token',
'net',
'openstack',
'registry',
'rhv',
'satellite6',
'scm',
'ssh',
'terraform',
'thycotic_dsv',
'thycotic_tss',
'vault',
'vmware',
]
assert sorted(CredentialType.defaults.keys()) == sorted(expected)
assert 'hashivault-kv-oidc' not in CredentialType.defaults
assert 'hashivault-ssh-oidc' not in CredentialType.defaults
assert sorted(CredentialType.defaults.keys()) == sorted(
[
'aim',
'aws',
'aws_secretsmanager_credential',
'azure_kv',
'azure_rm',
'bitbucket_dc_token',
'centrify_vault_kv',
'conjur',
'controller',
'galaxy_api_token',
'gce',
'github_token',
'github_app_lookup',
'gitlab_token',
'gpg_public_key',
'hashivault_kv',
'hashivault_ssh',
'hcp_terraform',
'insights',
'kubernetes_bearer_token',
'net',
'openstack',
'registry',
'rhv',
'satellite6',
'scm',
'ssh',
'terraform',
'thycotic_dsv',
'thycotic_tss',
'vault',
'vmware',
]
)
for type_ in CredentialType.defaults.values():
assert type_().managed is True
@pytest.mark.django_db
def test_default_cred_types_with_oidc_enabled():
from django.test import override_settings
from awx.main.models.credential import load_credentials, ManagedCredentialType
original_registry = ManagedCredentialType.registry.copy()
try:
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=True):
ManagedCredentialType.registry.clear()
load_credentials()
assert 'hashivault-kv-oidc' in CredentialType.defaults
assert 'hashivault-ssh-oidc' in CredentialType.defaults
finally:
ManagedCredentialType.registry = original_registry
@pytest.mark.django_db
def test_credential_creation(organization_factory):
org = organization_factory('test').organization

View File

@@ -8,7 +8,6 @@ from awx.main.models import (
Instance,
Host,
JobHostSummary,
Inventory,
InventoryUpdate,
InventorySource,
Project,
@@ -18,60 +17,14 @@ from awx.main.models import (
InstanceGroup,
Label,
ExecutionEnvironment,
Credential,
CredentialType,
CredentialInputSource,
Organization,
JobTemplate,
)
from awx.main.tasks import jobs
from awx.main.tasks.system import cluster_node_heartbeat
from awx.main.utils.db import bulk_update_sorted_by_id
from ansible_base.lib.testing.util import feature_flag_enabled, feature_flag_disabled
from django.db import OperationalError
from django.test.utils import override_settings
@pytest.fixture
def job_template_with_credentials():
"""
Factory fixture that creates a job template with specified credentials.
Usage:
job = job_template_with_credentials(ssh_cred, vault_cred)
"""
def _create_job_template(
*credentials, org_name='test-org', project_name='test-project', inventory_name='test-inventory', jt_name='test-jt', playbook='test.yml'
):
"""
Create a job template with the given credentials.
Args:
*credentials: Variable number of Credential objects to attach to the job template
org_name: Name for the organization
project_name: Name for the project
inventory_name: Name for the inventory
jt_name: Name for the job template
playbook: Playbook filename
Returns:
Job instance created from the job template
"""
org = Organization.objects.create(name=org_name)
proj = Project.objects.create(name=project_name, organization=org)
inv = Inventory.objects.create(name=inventory_name, organization=org)
jt = JobTemplate.objects.create(name=jt_name, project=proj, inventory=inv, playbook=playbook)
if credentials:
jt.credentials.add(*credentials)
return jt.create_unified_job()
return _create_job_template
@pytest.mark.django_db
def test_orphan_unified_job_creation(instance, inventory):
job = Job.objects.create(job_template=None, inventory=inventory, name='hi world')
@@ -309,442 +262,3 @@ class TestLaunchConfig:
assert config.execution_environment
# We just write the PK instead of trying to assign an item, that happens on the save
assert config.execution_environment_id == ee.id
@pytest.mark.django_db
def test_base_task_credentials_property(job_template_with_credentials):
"""Test that _credentials property caches credentials and doesn't re-query."""
task = jobs.RunJob()
# Create real credentials
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
vault_type = CredentialType.defaults['vault']()
vault_type.save()
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
vault_cred = Credential.objects.create(credential_type=vault_type, name='vault-cred')
# Create a job with credentials using fixture
job = job_template_with_credentials(ssh_cred, vault_cred)
task.instance = job
# First access should build credentials
result1 = task._credentials
assert len(result1) == 2
assert isinstance(result1, list)
# Second access should return cached value (we can verify by checking it's the same list object)
result2 = task._credentials
assert result2 is result1 # Same object reference
@pytest.mark.django_db
def test_run_job_machine_credential(job_template_with_credentials):
"""Test _machine_credential returns ssh credential from cache."""
task = jobs.RunJob()
# Create credentials
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
vault_type = CredentialType.defaults['vault']()
vault_type.save()
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
vault_cred = Credential.objects.create(credential_type=vault_type, name='vault-cred')
# Create a job using fixture
job = job_template_with_credentials(ssh_cred, vault_cred)
task.instance = job
# Set cached credentials
task._credentials = [ssh_cred, vault_cred]
# Get machine credential
result = task._machine_credential
assert result == ssh_cred
assert result.credential_type.kind == 'ssh'
@pytest.mark.django_db
def test_run_job_machine_credential_none(job_template_with_credentials):
"""Test _machine_credential returns None when no ssh credential exists."""
task = jobs.RunJob()
# Create only vault credential
vault_type = CredentialType.defaults['vault']()
vault_type.save()
vault_cred = Credential.objects.create(credential_type=vault_type, name='vault-cred')
job = job_template_with_credentials(vault_cred)
task.instance = job
# Set cached credentials
task._credentials = [vault_cred]
# Get machine credential
result = task._machine_credential
assert result is None
@pytest.mark.django_db
def test_run_job_vault_credentials(job_template_with_credentials):
"""Test _vault_credentials returns all vault credentials from cache."""
task = jobs.RunJob()
# Create credentials
vault_type = CredentialType.defaults['vault']()
vault_type.save()
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
vault_cred1 = Credential.objects.create(credential_type=vault_type, name='vault-1')
vault_cred2 = Credential.objects.create(credential_type=vault_type, name='vault-2')
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
job = job_template_with_credentials(vault_cred1, ssh_cred, vault_cred2)
task.instance = job
# Set cached credentials
task._credentials = [vault_cred1, ssh_cred, vault_cred2]
# Get vault credentials
result = task._vault_credentials
assert len(result) == 2
assert vault_cred1 in result
assert vault_cred2 in result
assert ssh_cred not in result
@pytest.mark.django_db
def test_run_job_network_credentials(job_template_with_credentials):
"""Test _network_credentials returns all network credentials from cache."""
task = jobs.RunJob()
# Create credentials
net_type = CredentialType.defaults['net']()
net_type.save()
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
net_cred = Credential.objects.create(credential_type=net_type, name='net-cred')
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
job = job_template_with_credentials(net_cred, ssh_cred)
task.instance = job
# Set cached credentials
task._credentials = [net_cred, ssh_cred]
# Get network credentials
result = task._network_credentials
assert len(result) == 1
assert result[0] == net_cred
@pytest.mark.django_db
def test_run_job_cloud_credentials(job_template_with_credentials):
"""Test _cloud_credentials returns all cloud credentials from cache."""
task = jobs.RunJob()
# Create credentials
aws_type = CredentialType.defaults['aws']()
aws_type.save()
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
aws_cred = Credential.objects.create(credential_type=aws_type, name='aws-cred')
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
job = job_template_with_credentials(aws_cred, ssh_cred)
task.instance = job
# Set cached credentials
task._credentials = [aws_cred, ssh_cred]
# Get cloud credentials
result = task._cloud_credentials
assert len(result) == 1
assert result[0] == aws_cred
@pytest.mark.django_db
@override_settings(RESOURCE_SERVER={'URL': 'https://gateway.example.com', 'SECRET_KEY': 'test-secret-key', 'VALIDATE_HTTPS': False})
def test_populate_workload_identity_tokens_with_flag_enabled(job_template_with_credentials, mocker):
"""Test populate_workload_identity_tokens sets context when flag is enabled."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
task = jobs.RunJob()
# Create credential types
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
# Create a workload identity credential type
hashivault_type = CredentialType(
name='HashiCorp Vault Secret Lookup (OIDC)',
kind='cloud',
managed=False,
inputs={
'fields': [
{'id': 'jwt_aud', 'type': 'string', 'label': 'JWT Audience'},
{'id': 'workload_identity_token', 'type': 'string', 'label': 'Workload Identity Token', 'secret': True, 'internal': True},
]
},
)
hashivault_type.save()
# Create credentials
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
source_cred = Credential.objects.create(credential_type=hashivault_type, name='vault-source', inputs={'jwt_aud': 'https://vault.example.com'})
target_cred = Credential.objects.create(credential_type=ssh_type, name='target-cred', inputs={'username': 'testuser'})
# Create input source linking source credential to target credential
input_source = CredentialInputSource.objects.create(
target_credential=target_cred, source_credential=source_cred, input_field_name='password', metadata={'path': 'secret/data/password'}
)
# Create a job using fixture
job = job_template_with_credentials(target_cred, ssh_cred)
task.instance = job
# Override cached_property so the loop uses these exact Python objects
task._credentials = [target_cred, ssh_cred]
# Mock only the HTTP response from the Gateway workload identity endpoint
mock_response = mocker.Mock(status_code=200)
mock_response.json.return_value = {'jwt': 'eyJ.test.jwt'}
mock_request = mocker.patch('requests.request', return_value=mock_response, autospec=True)
task.populate_workload_identity_tokens()
# Verify the HTTP call was made to the correct endpoint
mock_request.assert_called_once()
call_kwargs = mock_request.call_args.kwargs
assert call_kwargs['method'] == 'POST'
assert '/api/gateway/v1/workload_identity_tokens' in call_kwargs['url']
# Verify context was set on the credential, keyed by input source PK
assert input_source.pk in target_cred.context
assert target_cred.context[input_source.pk]['workload_identity_token'] == 'eyJ.test.jwt'
@pytest.mark.django_db
@override_settings(RESOURCE_SERVER={'URL': 'https://gateway.example.com', 'SECRET_KEY': 'test-secret-key', 'VALIDATE_HTTPS': False})
def test_populate_workload_identity_tokens_passes_workload_ttl_from_job_timeout(job_template_with_credentials, mocker):
"""Test populate_workload_identity_tokens passes workload_ttl_seconds from get_instance_timeout to the client."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
task = jobs.RunJob()
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
hashivault_type = CredentialType(
name='HashiCorp Vault Secret Lookup (OIDC)',
kind='cloud',
managed=False,
inputs={
'fields': [
{'id': 'jwt_aud', 'type': 'string', 'label': 'JWT Audience'},
{'id': 'workload_identity_token', 'type': 'string', 'label': 'Workload Identity Token', 'secret': True, 'internal': True},
]
},
)
hashivault_type.save()
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
source_cred = Credential.objects.create(credential_type=hashivault_type, name='vault-source', inputs={'jwt_aud': 'https://vault.example.com'})
target_cred = Credential.objects.create(credential_type=ssh_type, name='target-cred', inputs={'username': 'testuser'})
CredentialInputSource.objects.create(
target_credential=target_cred, source_credential=source_cred, input_field_name='password', metadata={'path': 'secret/data/password'}
)
job = job_template_with_credentials(target_cred, ssh_cred)
job.timeout = 3600
job.save()
task.instance = job
task._credentials = [target_cred, ssh_cred]
mock_response = mocker.Mock(status_code=200)
mock_response.json.return_value = {'jwt': 'eyJ.test.jwt'}
mock_request = mocker.patch('requests.request', return_value=mock_response, autospec=True)
task.populate_workload_identity_tokens()
call_kwargs = mock_request.call_args.kwargs
assert call_kwargs['method'] == 'POST'
json_body = call_kwargs.get('json', {})
assert json_body.get('workload_ttl_seconds') == 3600
@pytest.mark.django_db
def test_populate_workload_identity_tokens_with_flag_disabled(job_template_with_credentials):
"""Test populate_workload_identity_tokens sets error status when flag is disabled."""
with feature_flag_disabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
task = jobs.RunJob()
# Create credential types
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
# Create a workload identity credential type
hashivault_type = CredentialType(
name='HashiCorp Vault Secret Lookup (OIDC)',
kind='cloud',
managed=False,
inputs={
'fields': [
{'id': 'jwt_aud', 'type': 'string', 'label': 'JWT Audience'},
{'id': 'workload_identity_token', 'type': 'string', 'label': 'Workload Identity Token', 'secret': True, 'internal': True},
]
},
)
hashivault_type.save()
# Create credentials
source_cred = Credential.objects.create(credential_type=hashivault_type, name='vault-source')
target_cred = Credential.objects.create(credential_type=ssh_type, name='target-cred', inputs={'username': 'testuser'})
# Create input source linking source credential to target credential
# Note: Creates the relationship that will trigger the feature flag check
CredentialInputSource.objects.create(
target_credential=target_cred, source_credential=source_cred, input_field_name='password', metadata={'path': 'secret/data/password'}
)
# Create a job using fixture
job = job_template_with_credentials(target_cred)
task.instance = job
# Set cached credentials
task._credentials = [target_cred]
task.populate_workload_identity_tokens()
# Verify job status was set to error
job.refresh_from_db()
assert job.status == 'error'
assert 'FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED' in job.job_explanation
assert 'vault-source' in job.job_explanation
@pytest.mark.django_db
@override_settings(RESOURCE_SERVER={'URL': 'https://gateway.example.com', 'SECRET_KEY': 'test-secret-key', 'VALIDATE_HTTPS': False})
def test_populate_workload_identity_tokens_multiple_input_sources_per_credential(job_template_with_credentials, mocker):
"""Test that a single credential with two input sources from different workload identity
credential types gets a separate JWT token for each input source, keyed by input source PK."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
task = jobs.RunJob()
# Create credential types
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
# Create two different workload identity credential types
hashivault_kv_type = CredentialType(
name='HashiCorp Vault Secret Lookup (OIDC)',
kind='cloud',
managed=False,
inputs={
'fields': [
{'id': 'jwt_aud', 'type': 'string', 'label': 'JWT Audience'},
{'id': 'workload_identity_token', 'type': 'string', 'label': 'Workload Identity Token', 'secret': True, 'internal': True},
]
},
)
hashivault_kv_type.save()
hashivault_ssh_type = CredentialType(
name='HashiCorp Vault Signed SSH (OIDC)',
kind='cloud',
managed=False,
inputs={
'fields': [
{'id': 'jwt_aud', 'type': 'string', 'label': 'JWT Audience'},
{'id': 'workload_identity_token', 'type': 'string', 'label': 'Workload Identity Token', 'secret': True, 'internal': True},
]
},
)
hashivault_ssh_type.save()
# Create source credentials with different audiences
source_cred_kv = Credential.objects.create(
credential_type=hashivault_kv_type, name='vault-kv-source', inputs={'jwt_aud': 'https://vault-kv.example.com'}
)
source_cred_ssh = Credential.objects.create(
credential_type=hashivault_ssh_type, name='vault-ssh-source', inputs={'jwt_aud': 'https://vault-ssh.example.com'}
)
# Create target credential that uses both sources for different fields
target_cred = Credential.objects.create(credential_type=ssh_type, name='target-cred', inputs={'username': 'testuser'})
# Create two input sources on the same target credential, each for a different field
input_source_password = CredentialInputSource.objects.create(
target_credential=target_cred, source_credential=source_cred_kv, input_field_name='password', metadata={'path': 'secret/data/password'}
)
input_source_ssh_key = CredentialInputSource.objects.create(
target_credential=target_cred, source_credential=source_cred_ssh, input_field_name='ssh_key_data', metadata={'path': 'secret/data/ssh_key'}
)
# Create a job using fixture
job = job_template_with_credentials(target_cred)
task.instance = job
# Override cached_property so the loop uses this exact Python object
task._credentials = [target_cred]
# Mock HTTP responses - return different JWTs for each call
response_kv = mocker.Mock(status_code=200)
response_kv.json.return_value = {'jwt': 'eyJ.kv.jwt'}
response_ssh = mocker.Mock(status_code=200)
response_ssh.json.return_value = {'jwt': 'eyJ.ssh.jwt'}
mock_request = mocker.patch('requests.request', side_effect=[response_kv, response_ssh], autospec=True)
task.populate_workload_identity_tokens()
# Verify two separate HTTP calls were made (one per input source)
assert mock_request.call_count == 2
# Verify each call used the correct audience from its source credential
audiences_requested = {call.kwargs.get('json', {}).get('audience', '') for call in mock_request.call_args_list}
assert 'https://vault-kv.example.com' in audiences_requested
assert 'https://vault-ssh.example.com' in audiences_requested
# Verify context on the target credential has both tokens, keyed by input source PK
assert input_source_password.pk in target_cred.context
assert input_source_ssh_key.pk in target_cred.context
assert target_cred.context[input_source_password.pk]['workload_identity_token'] == 'eyJ.kv.jwt'
assert target_cred.context[input_source_ssh_key.pk]['workload_identity_token'] == 'eyJ.ssh.jwt'
@pytest.mark.django_db
def test_populate_workload_identity_tokens_without_workload_identity_credentials(job_template_with_credentials, mocker):
"""Test populate_workload_identity_tokens does nothing when no workload identity credentials."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
task = jobs.RunJob()
# Create only standard credentials (no workload identity)
ssh_type = CredentialType.defaults['ssh']()
ssh_type.save()
vault_type = CredentialType.defaults['vault']()
vault_type.save()
ssh_cred = Credential.objects.create(credential_type=ssh_type, name='ssh-cred')
vault_cred = Credential.objects.create(credential_type=vault_type, name='vault-cred')
# Create a job using fixture
job = job_template_with_credentials(ssh_cred, vault_cred)
task.instance = job
# Set cached credentials
task._credentials = [ssh_cred, vault_cred]
mocker.patch('awx.main.tasks.jobs.populate_claims_for_workload', return_value={'job_id': 123}, autospec=True)
task.populate_workload_identity_tokens()
# Verify no context was set
assert not hasattr(ssh_cred, '_context') or ssh_cred.context == {}
assert not hasattr(vault_cred, '_context') or vault_cred.context == {}

View File

@@ -18,14 +18,13 @@ from awx.main.tests.functional.conftest import * # noqa
from awx.main.tests.conftest import load_all_credentials # noqa: F401; pylint: disable=unused-import
from awx.main.tests import data
from awx.main.models import Project, JobTemplate, Organization, Inventory, WorkflowJob, UnifiedJob
from awx.main.models import Project, JobTemplate, Organization, Inventory
from awx.main.tasks.system import clear_setting_cache
logger = logging.getLogger(__name__)
PROJ_DATA = os.path.join(os.path.dirname(data.__file__), 'projects')
COLL_DATA = os.path.join(os.path.dirname(data.__file__), 'collections')
def _copy_folders(source_path, dest_path, clear=False):
@@ -57,7 +56,6 @@ def live_tmp_folder():
shutil.rmtree(path)
os.mkdir(path)
_copy_folders(PROJ_DATA, path)
_copy_folders(COLL_DATA, path)
for dirname in os.listdir(path):
source_dir = os.path.join(path, dirname)
subprocess.run(GIT_COMMANDS, cwd=source_dir, shell=True)
@@ -102,21 +100,6 @@ def wait_for_events(uj, timeout=2):
def unified_job_stdout(uj):
if type(uj) is UnifiedJob:
uj = uj.get_real_instance()
if isinstance(uj, WorkflowJob):
outputs = []
for node in uj.workflow_job_nodes.all().select_related('job').order_by('id'):
if node.job is None:
continue
outputs.append(
'workflow node {node_id} job {job_id} output:\n{output}'.format(
node_id=node.id,
job_id=node.job.id,
output=unified_job_stdout(node.job),
)
)
return '\n'.join(outputs)
wait_for_events(uj)
return '\n'.join([event.stdout for event in uj.get_event_queryset().order_by('created')])

View File

@@ -1,351 +0,0 @@
"""Tests for concurrent fact caching with --limit.
Reproduces bugs where concurrent jobs targeting different hosts via --limit
incorrectly modify (clear or revert) facts for hosts outside their limit.
Customer report: concurrent jobs on the same job template with different limits
cause facts set by an earlier-finishing job to be rolled back when the
later-finishing job completes.
See: https://github.com/jritter/concurrent-aap-fact-caching
Generated by Claude Opus 4.6 (claude-opus-4-6).
"""
import logging
import pytest
from django.utils.timezone import now
from awx.api.versioning import reverse
from awx.main.models import Inventory, JobTemplate
from awx.main.tests.live.tests.conftest import wait_for_job, wait_to_leave_status
logger = logging.getLogger(__name__)
@pytest.fixture
def concurrent_facts_inventory(default_org):
"""Inventory with two hosts for concurrent fact cache testing."""
inv_name = 'test_concurrent_fact_cache'
Inventory.objects.filter(organization=default_org, name=inv_name).delete()
inv = Inventory.objects.create(organization=default_org, name=inv_name)
inv.hosts.create(name='cc_host_0')
inv.hosts.create(name='cc_host_1')
return inv
@pytest.fixture
def concurrent_facts_jt(concurrent_facts_inventory, live_tmp_folder, post, admin, project_factory):
"""Job template configured for concurrent fact-cached runs."""
proj = project_factory(scm_url=f'file://{live_tmp_folder}/facts')
if proj.current_job:
wait_for_job(proj.current_job)
assert 'gather_slow.yml' in proj.playbooks, f'gather_slow.yml not in {proj.playbooks}'
jt_name = 'test_concurrent_fact_cache JT'
existing_jt = JobTemplate.objects.filter(name=jt_name).first()
if existing_jt:
existing_jt.delete()
result = post(
reverse('api:job_template_list'),
{
'name': jt_name,
'project': proj.id,
'playbook': 'gather_slow.yml',
'inventory': concurrent_facts_inventory.id,
'use_fact_cache': True,
'allow_simultaneous': True,
},
admin,
expect=201,
)
return JobTemplate.objects.get(id=result.data['id'])
def test_concurrent_limit_does_not_clear_facts(concurrent_facts_inventory, concurrent_facts_jt):
"""Concurrent jobs with different --limit must not clear each other's facts.
Generated by Claude Opus 4.6 (claude-opus-4-6).
Scenario:
- Inventory has cc_host_0 and cc_host_1, neither has prior facts
- Job A runs gather_slow.yml with limit=cc_host_0
- While Job A is still running (sleeping), Job B launches with limit=cc_host_1
- Both jobs set cacheable facts, but only for their respective limited host
- After both complete, BOTH hosts should have populated facts
The bug: get_hosts_for_fact_cache() returns ALL inventory hosts regardless
of --limit. start_fact_cache records them all in hosts_cached but writes
no fact files (no prior facts). When the later-finishing job runs
finish_fact_cache, it sees a missing fact file for the other job's host
and clears that host's facts.
"""
inv = concurrent_facts_inventory
jt = concurrent_facts_jt
# Launch Job A targeting cc_host_0
job_a = jt.create_unified_job()
job_a.limit = 'cc_host_0'
job_a.save(update_fields=['limit'])
job_a.signal_start()
# Wait for Job A to reach running (it will sleep inside the playbook)
wait_to_leave_status(job_a, 'pending')
wait_to_leave_status(job_a, 'waiting')
logger.info(f'Job A (id={job_a.id}) is now running with limit=cc_host_0')
# Launch Job B targeting cc_host_1 while Job A is still running
job_b = jt.create_unified_job()
job_b.limit = 'cc_host_1'
job_b.save(update_fields=['limit'])
job_b.signal_start()
# Verify that Job A is still running when Job B starts,
# otherwise the overlap that triggers the bug did not happen.
wait_to_leave_status(job_b, 'pending')
wait_to_leave_status(job_b, 'waiting')
job_a.refresh_from_db()
if job_a.status != 'running':
pytest.skip('Job A finished before Job B started running; overlap did not occur')
logger.info(f'Job B (id={job_b.id}) is now running with limit=cc_host_1 (concurrent with Job A)')
# Wait for both to complete
wait_for_job(job_a)
wait_for_job(job_b)
# Verify facts survived concurrent execution
host_0 = inv.hosts.get(name='cc_host_0')
host_1 = inv.hosts.get(name='cc_host_1')
# sanity
job_a.refresh_from_db()
job_b.refresh_from_db()
assert job_a.limit == "cc_host_0"
assert job_b.limit == "cc_host_1"
discovered_foos = [host_0.ansible_facts.get('foo'), host_1.ansible_facts.get('foo')]
assert discovered_foos == ['bar'] * 2, f'Unexpected facts on cc_host_0 or _1: {discovered_foos} after job a,b {job_a.id}, {job_b.id}'
def test_concurrent_limit_does_not_revert_facts(live_tmp_folder, run_job_from_playbook, concurrent_facts_inventory):
"""Concurrent jobs must not revert facts that a prior concurrent job just set.
Generated by Claude Opus 4.6 (claude-opus-4-6).
Scenario:
- First, populate both hosts with initial facts (foo=bar) via a
non-concurrent gather run
- Then run two concurrent jobs with different limits, each setting
a new value (foo=bar_v2 via extra_vars)
- After both complete, BOTH hosts should have foo=bar_v2
The bug: start_fact_cache writes the OLD facts (foo=bar) into each job's
artifact dir for ALL hosts. If ansible's cache plugin rewrites a non-limited
host's fact file with the stale content (updating the mtime), finish_fact_cache
treats it as a legitimate update and overwrites the DB with old values.
"""
# --- Seed both hosts with initial facts via a non-concurrent run ---
inv = concurrent_facts_inventory
scm_url = f'file://{live_tmp_folder}/facts'
res = run_job_from_playbook(
'seed_facts_for_revert_test',
'gather_slow.yml',
scm_url=scm_url,
jt_params={'use_fact_cache': True, 'allow_simultaneous': True, 'inventory': inv.id},
)
for host in inv.hosts.all():
assert host.ansible_facts.get('foo') == 'bar', f'Seed run failed to set facts on {host.name}: {host.ansible_facts}'
job = res['job']
wait_for_job(job)
# sanity, jobs should be set up to both have facts with just bar
host_0 = inv.hosts.get(name='cc_host_0')
host_1 = inv.hosts.get(name='cc_host_1')
discovered_foos = [host_0.ansible_facts.get('foo'), host_1.ansible_facts.get('foo')]
assert discovered_foos == ['bar'] * 2, f'Facts did not get expected initial values: {discovered_foos}'
jt = job.job_template
assert jt.allow_simultaneous is True
assert jt.use_fact_cache is True
# Sanity assertion, sometimes this would give problems from the Django rel cache
assert jt.project
# --- Run two concurrent jobs that write a new value ---
# Update the JT to pass extra_vars that change the fact value
jt.extra_vars = '{"extra_value": "_v2"}'
jt.save(update_fields=['extra_vars'])
job_a = jt.create_unified_job()
job_a.limit = 'cc_host_0'
job_a.save(update_fields=['limit'])
job_a.signal_start()
wait_to_leave_status(job_a, 'pending')
wait_to_leave_status(job_a, 'waiting')
job_b = jt.create_unified_job()
job_b.limit = 'cc_host_1'
job_b.save(update_fields=['limit'])
job_b.signal_start()
wait_to_leave_status(job_b, 'pending')
wait_to_leave_status(job_b, 'waiting')
job_a.refresh_from_db()
if job_a.status != 'running':
pytest.skip('Job A finished before Job B started running; overlap did not occur')
wait_for_job(job_a)
wait_for_job(job_b)
host_0 = inv.hosts.get(name='cc_host_0')
host_1 = inv.hosts.get(name='cc_host_1')
# Both hosts should have the UPDATED value, not the old seed value
discovered_foos = [host_0.ansible_facts.get('foo'), host_1.ansible_facts.get('foo')]
assert discovered_foos == ['bar_v2'] * 2, f'Facts were reverted to stale values by concurrent job cc_host_0 or cc_host_1: {discovered_foos}'
def test_fact_cache_scoped_to_inventory(live_tmp_folder, default_org, run_job_from_playbook):
"""finish_fact_cache must not modify hosts in other inventories.
Generated by Claude Opus 4.6 (claude-opus-4-6).
Bug: finish_fact_cache queries Host.objects.filter(name__in=host_names)
without an inventory_id filter, so hosts with the same name in different
inventories get their facts cross-contaminated.
"""
shared_name = 'scope_shared_host'
# Prepare for test by deleting junk from last run
for inv_name in ('test_fact_scope_inv1', 'test_fact_scope_inv2'):
inv = Inventory.objects.filter(name=inv_name).first()
if inv:
inv.delete()
inv1 = Inventory.objects.create(organization=default_org, name='test_fact_scope_inv1')
inv1.hosts.create(name=shared_name)
inv2 = Inventory.objects.create(organization=default_org, name='test_fact_scope_inv2')
host2 = inv2.hosts.create(name=shared_name)
# Give inv2's host distinct facts that should not be touched
original_facts = {'source': 'inventory_2', 'untouched': True}
host2.ansible_facts = original_facts
host2.ansible_facts_modified = now()
host2.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
# Run a fact-gathering job against inv1 only
run_job_from_playbook(
'test_fact_scope',
'gather.yml',
scm_url=f'file://{live_tmp_folder}/facts',
jt_params={'use_fact_cache': True, 'inventory': inv1.id},
)
# inv1's host should have facts
host1 = inv1.hosts.get(name=shared_name)
assert host1.ansible_facts, f'inv1 host should have facts after gather: {host1.ansible_facts}'
# inv2's host must NOT have been touched
host2.refresh_from_db()
assert host2.ansible_facts == original_facts, (
f'Host in a different inventory was modified by a fact cache operation '
f'on another inventory sharing the same hostname. '
f'Expected {original_facts!r}, got {host2.ansible_facts!r}'
)
def test_constructed_inventory_facts_saved_to_source_host(live_tmp_folder, default_org, run_job_from_playbook):
"""Facts from a constructed inventory job must be saved to the source host.
Generated by Claude Opus 4.6 (claude-opus-4-6).
Constructed inventories contain hosts that are references (via instance_id)
to 'real' hosts in input inventories. start_fact_cache correctly resolves
source hosts via get_hosts_for_fact_cache(), but finish_fact_cache must also
write facts back to the source hosts, not the constructed inventory's copies.
Scenario:
- Two input inventories each have a host named 'ci_shared_host'
- A constructed inventory uses both as inputs
- The inventory sync picks one source host (via instance_id) for the
constructed host — which one depends on input processing order
- Both source hosts start with distinct pre-existing facts
- A fact-gathering job runs against the constructed inventory
- After completion, the targeted source host should have the job's facts
- The OTHER source host must retain its original facts untouched
- The constructed host itself must NOT have facts stored on it
(constructed hosts are transient — recreated on each inventory sync)
"""
shared_name = 'ci_shared_host'
# Cleanup from prior runs
for inv_name in ('test_ci_facts_input1', 'test_ci_facts_input2', 'test_ci_facts_constructed'):
Inventory.objects.filter(name=inv_name).delete()
# --- Create two input inventories, each with an identically-named host ---
inv1 = Inventory.objects.create(organization=default_org, name='test_ci_facts_input1')
source_host1 = inv1.hosts.create(name=shared_name)
inv2 = Inventory.objects.create(organization=default_org, name='test_ci_facts_input2')
source_host2 = inv2.hosts.create(name=shared_name)
# Give both hosts distinct pre-existing facts so we can detect cross-contamination
host1_original_facts = {'source': 'inventory_1'}
source_host1.ansible_facts = host1_original_facts
source_host1.ansible_facts_modified = now()
source_host1.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
host2_original_facts = {'source': 'inventory_2'}
source_host2.ansible_facts = host2_original_facts
source_host2.ansible_facts_modified = now()
source_host2.save(update_fields=['ansible_facts', 'ansible_facts_modified'])
source_hosts_by_id = {source_host1.id: source_host1, source_host2.id: source_host2}
original_facts_by_id = {source_host1.id: host1_original_facts, source_host2.id: host2_original_facts}
# --- Create constructed inventory (sync will create hosts from inputs) ---
constructed_inv = Inventory.objects.create(
organization=default_org,
name='test_ci_facts_constructed',
kind='constructed',
)
constructed_inv.input_inventories.add(inv1)
constructed_inv.input_inventories.add(inv2)
# --- Run a fact-gathering job against the constructed inventory ---
# The job launch triggers an inventory sync which creates the constructed
# host with instance_id pointing to one of the source hosts.
scm_url = f'file://{live_tmp_folder}/facts'
run_job_from_playbook(
'test_ci_facts',
'gather.yml',
scm_url=scm_url,
jt_params={'use_fact_cache': True, 'inventory': constructed_inv.id},
)
# --- Determine which source host the constructed host points to ---
constructed_host = constructed_inv.hosts.get(name=shared_name)
target_id = int(constructed_host.instance_id)
other_id = (set(source_hosts_by_id.keys()) - {target_id}).pop()
target_host = source_hosts_by_id[target_id]
other_host = source_hosts_by_id[other_id]
target_host.refresh_from_db()
other_host.refresh_from_db()
constructed_host.refresh_from_db()
actual = [target_host.ansible_facts.get('foo'), other_host.ansible_facts, constructed_host.ansible_facts]
expected = ['bar', original_facts_by_id[other_id], {}]
assert actual == expected, (
f'Constructed inventory fact cache wrote to wrong host(s). '
f'target source host (id={target_id}) foo={actual[0]!r}, '
f'other source host (id={other_id}) facts={actual[1]!r}, '
f'constructed host facts={actual[2]!r}; expected {expected!r}'
)

View File

@@ -1,347 +0,0 @@
"""
Integration tests for external query file functionality (AAP-58470).
Tests verify the end-to-end external query file workflow for indirect node
counting using real AWX job execution. A fixture-created vendor collection
at /var/lib/awx/vendor_collections/ provides external query files, simulating
what the build-time (AAP-58426) and deployment (AAP-58557) integrations will
provide once available.
Test data:
- Collection 'demo.external' at various versions (no embedded query)
- External query files in mock redhat.indirect_accounting collection
"""
import os
import shutil
import time
import yaml
import pytest
from flags.state import enable_flag, disable_flag, flag_enabled
from awx.main.tests.live.tests.conftest import wait_for_events, unified_job_stdout
from awx.main.tasks.host_indirect import save_indirect_host_entries
from awx.main.models.indirect_managed_node_audit import IndirectManagedNodeAudit
from awx.main.models.event_query import EventQuery
from awx.main.models import Job
# --- Constants ---
EXTERNAL_QUERY_JQ = '{name: .name, canonical_facts: {host_name: .direct_host_name}, facts: {device_type: .device_type}}'
EXTERNAL_QUERY_CONTENT = yaml.dump(
{'demo.external.example': {'query': EXTERNAL_QUERY_JQ}},
default_flow_style=False,
)
# For precedence test: different jq (no device_type in facts) so we can detect which query was used
EXTERNAL_QUERY_FOR_DEMO_QUERY_JQ = '{name: .name, canonical_facts: {host_name: .direct_host_name}, facts: {}}'
EXTERNAL_QUERY_FOR_DEMO_QUERY_CONTENT = yaml.dump(
{'demo.query.example': {'query': EXTERNAL_QUERY_FOR_DEMO_QUERY_JQ}},
default_flow_style=False,
)
VENDOR_COLLECTIONS_BASE = '/var/lib/awx/vendor_collections'
# --- Fixtures ---
@pytest.fixture
def enable_indirect_host_counting():
"""Enable FEATURE_INDIRECT_NODE_COUNTING_ENABLED flag for the test.
Only creates a FlagState DB record if the flag isn't already enabled
(e.g. via development_defaults.py), to avoid UniqueViolation errors
and to avoid leaking state to other tests.
"""
flag_name = "FEATURE_INDIRECT_NODE_COUNTING_ENABLED"
was_enabled = flag_enabled(flag_name)
if not was_enabled:
enable_flag(flag_name)
yield
if not was_enabled:
disable_flag(flag_name)
@pytest.fixture
def vendor_collections_dir():
"""Set up mock redhat.indirect_accounting collection at /var/lib/awx/vendor_collections/.
Creates the collection structure with external query files:
- demo.external.1.0.0.yml (exact match for v1.0.0)
- demo.external.1.1.0.yml (fallback target for v1.5.0)
- demo.query.0.0.1.yml (for precedence test with embedded-query collection)
"""
base = os.path.join(VENDOR_COLLECTIONS_BASE, 'ansible_collections', 'redhat', 'indirect_accounting')
queries_path = os.path.join(base, 'extensions', 'audit', 'external_queries')
meta_path = os.path.join(base, 'meta')
os.makedirs(queries_path, exist_ok=True)
os.makedirs(meta_path, exist_ok=True)
# galaxy.yml for valid collection structure
with open(os.path.join(base, 'galaxy.yml'), 'w') as f:
yaml.dump(
{
'namespace': 'redhat',
'name': 'indirect_accounting',
'version': '1.0.0',
'description': 'Test fixture for external query integration tests',
'authors': ['AWX Tests'],
'dependencies': {},
},
f,
)
# meta/runtime.yml
with open(os.path.join(meta_path, 'runtime.yml'), 'w') as f:
yaml.dump({'requires_ansible': '>=2.15.0'}, f)
# External query files for demo.external collection
for version in ('1.0.0', '1.1.0'):
with open(os.path.join(queries_path, f'demo.external.{version}.yml'), 'w') as f:
f.write(EXTERNAL_QUERY_CONTENT)
# External query file for demo.query collection (precedence test)
with open(os.path.join(queries_path, 'demo.query.0.0.1.yml'), 'w') as f:
f.write(EXTERNAL_QUERY_FOR_DEMO_QUERY_CONTENT)
yield base
# Cleanup: only remove the collection we created, not the entire vendor root
shutil.rmtree(base, ignore_errors=True)
@pytest.fixture(autouse=True)
def cleanup_test_data():
"""Clean up EventQuery and IndirectManagedNodeAudit records after each test."""
yield
EventQuery.objects.filter(fqcn='demo.external').delete()
EventQuery.objects.filter(fqcn='demo.query').delete()
IndirectManagedNodeAudit.objects.filter(job__name__icontains='external_query').delete()
# --- Helpers ---
def run_external_query_job(run_job_from_playbook, live_tmp_folder, test_name, project_dir, jt_params=None):
"""Run a job and return the Job object after waiting for indirect host processing."""
scm_url = f'file://{live_tmp_folder}/{project_dir}'
run_job_from_playbook(test_name, 'run_task.yml', scm_url=scm_url, jt_params=jt_params)
job = Job.objects.filter(name__icontains=test_name).order_by('-created').first()
assert job is not None, f'Job not found for test {test_name}'
wait_for_events(job)
return job
def wait_for_indirect_processing(job, expect_records=True, timeout=5):
"""Wait for indirect host processing to complete.
Follows the same pattern as test_indirect_host_counting.py:53-72.
"""
# Ensure indirect host processing runs (wait_for_events already called by caller)
job.refresh_from_db()
if job.event_queries_processed is False:
save_indirect_host_entries.delay(job.id, wait_for_events=False)
if expect_records:
# Poll for audit records to appear
for _ in range(20):
if IndirectManagedNodeAudit.objects.filter(job=job).exists():
break
time.sleep(0.25)
else:
raise RuntimeError(f'No IndirectManagedNodeAudit records populated for job_id={job.id}')
else:
# For negative tests, wait a reasonable time to confirm no records appear
time.sleep(timeout)
job.refresh_from_db()
# --- AC8.1: External query populates IndirectManagedNodeAudit correctly ---
def test_external_query_populates_audit_table(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.1: Job using demo.external.example with external query file populates
IndirectManagedNodeAudit table correctly.
Uses demo.external v1.0.0 with exact-match external query file demo.external.1.0.0.yml.
"""
job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_1',
'test_host_query_external_v1_0_0',
)
wait_for_indirect_processing(job, expect_records=True)
# Verify installed_collections captured demo.external
assert 'demo.external' in job.installed_collections
assert 'host_query' in job.installed_collections['demo.external']
# Verify IndirectManagedNodeAudit records
assert IndirectManagedNodeAudit.objects.filter(job=job).count() == 1
host_audit = IndirectManagedNodeAudit.objects.filter(job=job).first()
assert host_audit.canonical_facts == {'host_name': 'foo_host_default'}
assert host_audit.facts == {'device_type': 'Fake Host'}
assert host_audit.name == 'vm-foo'
assert host_audit.organization == job.organization
assert 'demo.external.example' in host_audit.events
# --- AC8.2: Precedence - embedded query takes precedence over external ---
def test_embedded_query_takes_precedence(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.2: When collection has both embedded and external query files,
the embedded query takes precedence.
Uses demo.query v0.0.1 which HAS an embedded query (extensions/audit/event_query.yml).
An external query (demo.query.0.0.1.yml) also exists but uses a different jq expression
(no device_type in facts). By checking the audit record's facts, we verify which query was used.
"""
# Run with demo.query collection (has embedded query)
job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_2',
'test_host_query',
)
wait_for_indirect_processing(job, expect_records=True)
# Verify the embedded query was used (includes device_type in facts)
host_audit = IndirectManagedNodeAudit.objects.filter(job=job).first()
assert host_audit.facts == {'device_type': 'Fake Host'}, (
'Expected embedded query output (with device_type). ' 'If facts is {}, the external query was incorrectly used instead.'
)
# --- AC8.3: Version fallback to compatible version ---
def test_fallback_to_compatible_version(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.3: Job using collection version with no exact query file falls back
correctly to compatible version.
Uses demo.external v1.5.0. No demo.external.1.5.0.yml exists, but
demo.external.1.1.0.yml is available (same major version, highest <= 1.5.0).
The fallback should find and use the 1.1.0 query.
"""
job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_3',
'test_host_query_external_v1_5_0',
)
wait_for_indirect_processing(job, expect_records=True)
# Verify installed_collections captured demo.external at v1.5.0
assert 'demo.external' in job.installed_collections
assert job.installed_collections['demo.external']['version'] == '1.5.0'
# Verify IndirectManagedNodeAudit records were created via fallback
assert IndirectManagedNodeAudit.objects.filter(job=job).count() == 1
host_audit = IndirectManagedNodeAudit.objects.filter(job=job).first()
assert host_audit.canonical_facts == {'host_name': 'foo_host_default'}
assert host_audit.facts == {'device_type': 'Fake Host'}
assert host_audit.name == 'vm-foo'
# --- AC8.4: Fallback queries don't overcount ---
def test_fallback_does_not_overcount(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.4: Fallback queries don't count MORE nodes than exact-version queries.
Runs two jobs:
1. Exact match scenario (demo.external v1.0.0 -> demo.external.1.0.0.yml)
2. Fallback scenario (demo.external v1.5.0 -> falls back to demo.external.1.1.0.yml)
Verifies that fallback record count <= exact record count.
"""
# Run exact-match job
exact_job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_4_exact',
'test_host_query_external_v1_0_0',
)
wait_for_indirect_processing(exact_job, expect_records=True)
exact_count = IndirectManagedNodeAudit.objects.filter(job=exact_job).count()
# Run fallback job
fallback_job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_4_fallback',
'test_host_query_external_v1_5_0',
)
wait_for_indirect_processing(fallback_job, expect_records=True)
fallback_count = IndirectManagedNodeAudit.objects.filter(job=fallback_job).count()
# Critical safety check: fallback must never count MORE than exact
assert fallback_count <= exact_count, (
f'Overcounting detected! Fallback produced {fallback_count} records ' f'but exact match produced only {exact_count} records.'
)
# Both use the same jq expression and same module, so counts should be equal
assert exact_count == fallback_count
# --- AC8.5: Warning logs contain correct version information ---
def test_fallback_log_contains_version_info(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.5: Warning logs contain correct version information when fallback is used.
Runs a job with verbosity=1 so callback plugin verbose output is captured.
Verifies the log contains the installed version (1.5.0), fallback version (1.1.0),
and collection FQCN (demo.external).
"""
job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_5',
'test_host_query_external_v1_5_0',
jt_params={'verbosity': 1},
)
wait_for_indirect_processing(job, expect_records=True)
# Get job stdout to check for fallback log message
stdout = unified_job_stdout(job)
# The callback plugin emits: "Using external query {version_used} for {fqcn} v{ver}."
assert '1.1.0' in stdout, f'Fallback version 1.1.0 not found in job stdout. stdout:\n{stdout}'
assert 'demo.external' in stdout, f'Collection FQCN demo.external not found in job stdout. stdout:\n{stdout}'
assert '1.5.0' in stdout, f'Installed version 1.5.0 not found in job stdout. stdout:\n{stdout}'
# --- AC8.6: No counting when no compatible fallback exists ---
def test_no_counting_without_compatible_fallback(live_tmp_folder, run_job_from_playbook, enable_indirect_host_counting, vendor_collections_dir):
"""AC8.6: No counting occurs when no compatible fallback exists.
Uses demo.external v3.0.0 with only v1.x external query files available.
Since major versions differ (3 vs 1), no fallback should occur and no
IndirectManagedNodeAudit records should be created.
"""
job = run_external_query_job(
run_job_from_playbook,
live_tmp_folder,
'external_query_ac8_6',
'test_host_query_external_v3_0_0',
)
wait_for_indirect_processing(job, expect_records=False)
# No audit records should exist for this job
assert IndirectManagedNodeAudit.objects.filter(job=job).count() == 0, (
'IndirectManagedNodeAudit records were created despite no compatible ' 'fallback existing for demo.external v3.0.0 (only v1.x queries available).'
)

View File

@@ -47,34 +47,3 @@ def test__get_credential_type_class_invalid_params():
assert type(e.value) is ValueError
assert str(e.value) == 'Expected only apps or app_config to be defined, not both'
def test_credential_context_property():
"""Test that credential context property initializes empty dict and persists across accesses."""
ct = CredentialType(name='Test Cred', kind='vault')
cred = Credential(id=1, name='Test Credential', credential_type=ct, inputs={})
# First access should return empty dict
context = cred.context
assert context == {}
# Modify the context
context['test_key'] = 'test_value'
# Second access should return the same dict with modifications
assert cred.context == {'test_key': 'test_value'}
assert cred.context is context # Same object reference
def test_credential_context_property_independent_instances():
"""Test that context property is independent between credential instances."""
ct = CredentialType(name='Test Cred', kind='vault')
cred1 = Credential(id=1, name='Cred 1', credential_type=ct, inputs={})
cred2 = Credential(id=2, name='Cred 2', credential_type=ct, inputs={})
cred1.context['key1'] = 'value1'
cred2.context['key2'] = 'value2'
assert cred1.context == {'key1': 'value1'}
assert cred2.context == {'key2': 'value2'}
assert cred1.context is not cred2.context

View File

@@ -2,7 +2,6 @@
import json
import os
import pytest
from unittest import mock
from awx.main.models import (
Inventory,
@@ -100,55 +99,52 @@ def test_start_job_fact_cache_within_timeout(hosts, tmpdir):
def test_finish_job_fact_cache_clear(hosts, mocker, ref_time, tmpdir):
artifacts_dir = str(tmpdir.mkdir("artifacts"))
inventory_id = 5
start_fact_cache(hosts, artifacts_dir=artifacts_dir, timeout=0, inventory_id=inventory_id)
mocker.patch('awx.main.tasks.facts.bulk_update_sorted_by_id')
# Remove the fact file for hosts[1] to simulate ansible's clear_facts
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
os.remove(os.path.join(fact_cache_dir, hosts[1].name))
hosts_qs = mock.MagicMock()
hosts_qs.filter.return_value.order_by.return_value.iterator.return_value = iter(hosts)
finish_fact_cache(hosts_qs, artifacts_dir=artifacts_dir, inventory_id=inventory_id)
# hosts[1] should have had its facts cleared (file was missing, job_created=None)
assert hosts[1].ansible_facts == {}
assert hosts[1].ansible_facts_modified > ref_time
# Other hosts should be unmodified (fact files exist but weren't changed by ansible)
for host in (hosts[0], hosts[2], hosts[3]):
assert host.ansible_facts == {"a": 1, "b": 2}
assert host.ansible_facts_modified == ref_time
def test_finish_job_fact_cache_with_bad_data(hosts, mocker, tmpdir):
artifacts_dir = str(tmpdir.mkdir("artifacts"))
inventory_id = 5
start_fact_cache(hosts, artifacts_dir=artifacts_dir, timeout=0, inventory_id=inventory_id)
fact_cache = os.path.join(tmpdir, 'facts')
start_fact_cache(hosts, fact_cache, timeout=0)
bulk_update = mocker.patch('awx.main.tasks.facts.bulk_update_sorted_by_id')
# Overwrite fact files with invalid JSON and set future mtime
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
# Mock the os.path.exists behavior for host deletion
# Let's assume the fact file for hosts[1] is missing.
mocker.patch('os.path.exists', side_effect=lambda path: hosts[1].name not in path)
# Simulate one host's fact file getting deleted manually
host_to_delete_filepath = os.path.join(fact_cache, hosts[1].name)
# Simulate the file being removed by checking existence first, to avoid FileNotFoundError
if os.path.exists(host_to_delete_filepath):
os.remove(host_to_delete_filepath)
finish_fact_cache(fact_cache)
# Simulate side effects that would normally be applied during bulk update
hosts[1].ansible_facts = {}
hosts[1].ansible_facts_modified = now()
# Verify facts are preserved for hosts with valid cache files
for host in (hosts[0], hosts[2], hosts[3]):
assert host.ansible_facts == {"a": 1, "b": 2}
assert host.ansible_facts_modified == ref_time
assert hosts[1].ansible_facts_modified > ref_time
# Current implementation skips the call entirely if hosts_to_update == []
bulk_update.assert_not_called()
def test_finish_job_fact_cache_with_bad_data(hosts, mocker, tmpdir):
fact_cache = os.path.join(tmpdir, 'facts')
start_fact_cache(hosts, fact_cache, timeout=0)
bulk_update = mocker.patch('django.db.models.query.QuerySet.bulk_update')
for h in hosts:
filepath = os.path.join(fact_cache_dir, h.name)
filepath = os.path.join(fact_cache, h.name)
with open(filepath, 'w') as f:
f.write('not valid json!')
f.flush()
new_modification_time = time.time() + 3600
os.utime(filepath, (new_modification_time, new_modification_time))
hosts_qs = mock.MagicMock()
hosts_qs.filter.return_value.order_by.return_value.iterator.return_value = iter(hosts)
finish_fact_cache(fact_cache)
finish_fact_cache(hosts_qs, artifacts_dir=artifacts_dir, inventory_id=inventory_id)
# Invalid JSON should be skipped — no hosts updated
updated_hosts = bulk_update.call_args[0][1]
assert updated_hosts == []
bulk_update.assert_not_called()

View File

@@ -1,4 +1,8 @@
# -*- coding: utf-8 -*-
import os
import tempfile
import shutil
import pytest
from unittest import mock
@@ -28,58 +32,20 @@ from ansible_base.lib.workload_identity.controller import AutomationControllerJo
@pytest.fixture
def private_data_dir(tmp_path):
private_data = tmp_path / 'awx_pdd'
private_data.mkdir()
def private_data_dir():
private_data = tempfile.mkdtemp(prefix='awx_')
for subfolder in ('inventory', 'env'):
(private_data / subfolder).mkdir()
return str(private_data)
@pytest.fixture
def job_template_with_credentials():
"""
Factory fixture that creates a job template with specified credentials.
Usage:
job = job_template_with_credentials(ssh_cred, vault_cred)
"""
def _create_job_template(
*credentials, org_name='test-org', project_name='test-project', inventory_name='test-inventory', jt_name='test-jt', playbook='test.yml'
):
"""
Create a job template with the given credentials.
Args:
*credentials: Variable number of Credential objects to attach to the job template
org_name: Name for the organization
project_name: Name for the project
inventory_name: Name for the inventory
jt_name: Name for the job template
playbook: Playbook filename
Returns:
Job instance created from the job template
"""
org = Organization.objects.create(name=org_name)
proj = Project.objects.create(name=project_name, organization=org)
inv = Inventory.objects.create(name=inventory_name, organization=org)
jt = JobTemplate.objects.create(name=jt_name, project=proj, inventory=inv, playbook=playbook)
if credentials:
jt.credentials.add(*credentials)
return jt.create_unified_job()
return _create_job_template
runner_subfolder = os.path.join(private_data, subfolder)
os.makedirs(runner_subfolder, exist_ok=True)
yield private_data
shutil.rmtree(private_data, True)
@mock.patch('awx.main.tasks.facts.settings')
@mock.patch('awx.main.tasks.jobs.create_partition', return_value=True)
def test_pre_post_run_hook_facts(mock_create_partition, mock_facts_settings, private_data_dir, execution_environment):
# Create mocked inventory and host queryset
inventory = mock.MagicMock(spec=Inventory, pk=1, kind='')
inventory = mock.MagicMock(spec=Inventory, pk=1)
host1 = mock.MagicMock(spec=Host, id=1, name='host1', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=now(), inventory=inventory)
host2 = mock.MagicMock(spec=Host, id=2, name='host2', ansible_facts={"a": 1, "b": 2}, ansible_facts_modified=now(), inventory=inventory)
@@ -96,16 +62,12 @@ def test_pre_post_run_hook_facts(mock_create_partition, mock_facts_settings, pri
proj = mock.MagicMock(spec=Project, pk=1, organization=org)
job = mock.MagicMock(
spec=Job,
pk=1,
id=1,
use_fact_cache=True,
project=proj,
organization=org,
job_slice_number=1,
job_slice_count=1,
inventory=inventory,
inventory_id=inventory.pk,
created=now(),
execution_environment=execution_environment,
)
job.get_hosts_for_fact_cache = Job.get_hosts_for_fact_cache.__get__(job)
@@ -137,11 +99,9 @@ def test_pre_post_run_hook_facts(mock_create_partition, mock_facts_settings, pri
@mock.patch('awx.main.tasks.facts.bulk_update_sorted_by_id')
@mock.patch('awx.main.tasks.facts.settings')
@mock.patch('awx.main.tasks.jobs.create_partition', return_value=True)
def test_pre_post_run_hook_facts_deleted_sliced(
mock_create_partition, mock_facts_settings, mock_bulk_update_sorted_by_id, private_data_dir, execution_environment
):
def test_pre_post_run_hook_facts_deleted_sliced(mock_create_partition, mock_facts_settings, private_data_dir, execution_environment):
# Fully mocked inventory
mock_inventory = mock.MagicMock(spec=Inventory, pk=1, kind='')
mock_inventory = mock.MagicMock(spec=Inventory)
# Create 999 mocked Host instances
hosts = []
@@ -167,8 +127,6 @@ def test_pre_post_run_hook_facts_deleted_sliced(
# Mock job object
job = mock.MagicMock(spec=Job)
job.pk = 2
job.id = 2
job.use_fact_cache = True
job.project = proj
job.organization = org
@@ -176,8 +134,6 @@ def test_pre_post_run_hook_facts_deleted_sliced(
job.job_slice_count = 3
job.execution_environment = execution_environment
job.inventory = mock_inventory
job.inventory_id = mock_inventory.pk
job.created = now()
job.job_env.get.return_value = private_data_dir
# Bind actual method for host filtering
@@ -471,122 +427,3 @@ def test_populate_claims_for_adhoc_command(workload_attrs, expected_claims):
claims = jobs.populate_claims_for_workload(adhoc_command)
assert claims == expected_claims
@mock.patch('awx.main.tasks.jobs.get_workload_identity_client')
def test_retrieve_workload_identity_jwt_returns_jwt_from_client(mock_get_client):
"""retrieve_workload_identity_jwt returns the JWT string from the client."""
mock_client = mock.MagicMock()
mock_response = mock.MagicMock()
mock_response.jwt = 'eyJ.test.jwt'
mock_client.request_workload_jwt.return_value = mock_response
mock_get_client.return_value = mock_client
unified_job = Job()
unified_job.id = 42
unified_job.name = 'Test Job'
unified_job.launch_type = 'manual'
unified_job.organization = Organization(id=1, name='Test Org')
unified_job.unified_job_template = None
unified_job.instance_group = None
result = jobs.retrieve_workload_identity_jwt(unified_job, audience='https://api.example.com', scope='aap_controller_automation_job')
assert result == 'eyJ.test.jwt'
mock_client.request_workload_jwt.assert_called_once()
call_kwargs = mock_client.request_workload_jwt.call_args[1]
assert call_kwargs['audience'] == 'https://api.example.com'
assert call_kwargs['scope'] == 'aap_controller_automation_job'
assert 'claims' in call_kwargs
assert call_kwargs['claims'][AutomationControllerJobScope.CLAIM_JOB_ID] == 42
assert call_kwargs['claims'][AutomationControllerJobScope.CLAIM_JOB_NAME] == 'Test Job'
@mock.patch('awx.main.tasks.jobs.get_workload_identity_client')
def test_retrieve_workload_identity_jwt_passes_audience_and_scope(mock_get_client):
"""retrieve_workload_identity_jwt passes audience and scope to the client."""
mock_client = mock.MagicMock()
mock_client.request_workload_jwt.return_value = mock.MagicMock(jwt='token')
mock_get_client.return_value = mock_client
unified_job = mock.MagicMock()
audience = 'custom_audience'
scope = 'custom_scope'
with mock.patch('awx.main.tasks.jobs.populate_claims_for_workload', return_value={'job_id': 1}):
jobs.retrieve_workload_identity_jwt(unified_job, audience=audience, scope=scope)
mock_client.request_workload_jwt.assert_called_once_with(claims={'job_id': 1}, scope=scope, audience=audience)
@mock.patch('awx.main.tasks.jobs.get_workload_identity_client')
def test_retrieve_workload_identity_jwt_passes_workload_ttl(mock_get_client):
"""retrieve_workload_identity_jwt passes workload_ttl_seconds when provided."""
mock_client = mock.Mock()
mock_client.request_workload_jwt.return_value = mock.Mock(jwt='token')
mock_get_client.return_value = mock_client
unified_job = mock.MagicMock()
with mock.patch('awx.main.tasks.jobs.populate_claims_for_workload', return_value={'job_id': 1}):
jobs.retrieve_workload_identity_jwt(
unified_job,
audience='https://vault.example.com',
scope='aap_controller_automation_job',
workload_ttl_seconds=3600,
)
mock_client.request_workload_jwt.assert_called_once_with(
claims={'job_id': 1},
scope='aap_controller_automation_job',
audience='https://vault.example.com',
workload_ttl_seconds=3600,
)
@mock.patch('awx.main.tasks.jobs.get_workload_identity_client')
def test_retrieve_workload_identity_jwt_raises_when_client_not_configured(mock_get_client):
"""retrieve_workload_identity_jwt raises RuntimeError when client is None."""
mock_get_client.return_value = None
unified_job = mock.MagicMock()
with pytest.raises(RuntimeError, match="Workload identity client is not configured"):
jobs.retrieve_workload_identity_jwt(unified_job, audience='test_audience', scope='test_scope')
@pytest.mark.parametrize('effective_timeout,expected_ttl', [(3600, 3600), (0, None)])
@mock.patch('awx.main.tasks.jobs.retrieve_workload_identity_jwt')
@mock.patch('awx.main.tasks.jobs.flag_enabled', return_value=True)
def test_populate_workload_identity_tokens_passes_get_instance_timeout_to_client(mock_flag_enabled, mock_retrieve_jwt, effective_timeout, expected_ttl):
"""populate_workload_identity_tokens passes get_instance_timeout() value as workload_ttl_seconds to retrieve_workload_identity_jwt."""
mock_retrieve_jwt.return_value = 'eyJ.test.jwt'
task = jobs.RunJob()
task.instance = mock.MagicMock()
# Minimal credential with workload identity input source
credential_ctx = {}
input_src = mock.MagicMock()
input_src.pk = 1
input_src.source_credential = mock.MagicMock()
input_src.source_credential.get_input.return_value = 'https://vault.example.com'
input_src.source_credential.name = 'vault-cred'
input_src.source_credential.credential_type = mock.MagicMock()
input_src.source_credential.credential_type.inputs = {'fields': [{'id': 'workload_identity_token', 'internal': True}]}
credential = mock.MagicMock()
credential.context = credential_ctx
credential.input_sources = mock.MagicMock()
credential.input_sources.all.return_value = [input_src]
task._credentials = [credential]
with mock.patch.object(task, 'get_instance_timeout', return_value=effective_timeout):
task.populate_workload_identity_tokens()
mock_flag_enabled.assert_called_once_with("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED")
mock_retrieve_jwt.assert_called_once_with(
task.instance,
audience='https://vault.example.com',
scope=AutomationControllerJobScope.name,
workload_ttl_seconds=expected_ttl,
)

View File

@@ -76,9 +76,6 @@ def test_custom_error_messages(schema, given, message):
({'fields': [{'id': 'token', 'label': 'Token', 'secret': 'bad'}]}, False),
({'fields': [{'id': 'token', 'label': 'Token', 'ask_at_runtime': True}]}, True),
({'fields': [{'id': 'token', 'label': 'Token', 'ask_at_runtime': 'bad'}]}, False), # noqa
({'fields': [{'id': 'token', 'label': 'Token', 'internal': True}]}, True),
({'fields': [{'id': 'token', 'label': 'Token', 'internal': False}]}, True),
({'fields': [{'id': 'token', 'label': 'Token', 'internal': 'bad'}]}, False),
({'fields': [{'id': 'become_method', 'label': 'Become', 'choices': 'not-a-list'}]}, False), # noqa
({'fields': [{'id': 'become_method', 'label': 'Become', 'choices': []}]}, False),
({'fields': [{'id': 'become_method', 'label': 'Become', 'choices': ['su', 'sudo']}]}, True), # noqa
@@ -207,68 +204,6 @@ def test_credential_creation_validation_failure(inputs):
assert e.type in (ValidationError, DRFValidationError)
def test_credential_input_field_excludes_internal_fields():
"""Internal fields should be excluded from the schema generated by CredentialInputField,
preventing users from providing values for internally resolved fields."""
type_ = CredentialType(
kind='cloud',
name='SomeCloud',
managed=True,
inputs={
'fields': [
{'id': 'username', 'label': 'Username', 'type': 'string'},
{'id': 'resolved_token', 'label': 'Token', 'type': 'string', 'internal': True},
]
},
)
cred = Credential(credential_type=type_, name="Test Credential", inputs={'username': 'joe'})
field = cred._meta.get_field('inputs')
schema = field.schema(cred)
assert 'username' in schema['properties']
assert 'resolved_token' not in schema['properties']
def test_credential_input_field_rejects_values_for_internal_fields():
"""Users should not be able to provide values for fields marked as internal."""
type_ = CredentialType(
kind='cloud',
name='SomeCloud',
managed=True,
inputs={
'fields': [
{'id': 'username', 'label': 'Username', 'type': 'string'},
{'id': 'resolved_token', 'label': 'Token', 'type': 'string', 'internal': True},
]
},
)
cred = Credential(credential_type=type_, name="Test Credential", inputs={'username': 'joe', 'resolved_token': 'secret'})
field = cred._meta.get_field('inputs')
with pytest.raises(Exception) as e:
field.validate(cred.inputs, cred)
assert e.type in (ValidationError, DRFValidationError)
def test_credential_input_field_accepts_non_internal_fields_only():
"""Credentials with only non-internal field values should validate successfully."""
type_ = CredentialType(
kind='cloud',
name='SomeCloud',
managed=True,
inputs={
'fields': [
{'id': 'username', 'label': 'Username', 'type': 'string'},
{'id': 'resolved_token', 'label': 'Token', 'type': 'string', 'internal': True},
]
},
)
cred = Credential(credential_type=type_, name="Test Credential", inputs={'username': 'joe'})
field = cred._meta.get_field('inputs')
# Should not raise
field.validate(cred.inputs, cred)
def test_implicit_role_field_parents():
"""This assures that every ImplicitRoleField only references parents
which are relationships that actually exist

View File

@@ -1,431 +0,0 @@
"""
Unit tests for external query discovery and version fallback logic.
Tests for AAP-58456: Unit Test Suite for External Query Handling
"""
import sys
from io import StringIO
from unittest import mock
import pytest
from packaging.version import Version
# Helper for mocking importlib.resources.files() path traversal
def create_chainable_path_mock(final_mock, depth=3):
"""Mock that supports chained / operations: mock / 'a' / 'b' / 'c' -> final_mock"""
class ChainableMock:
def __init__(self, d=0):
self.d = d
def __truediv__(self, other):
return final_mock if self.d >= depth - 1 else ChainableMock(self.d + 1)
return ChainableMock()
def create_queries_dir_mock(file_lookup_func):
"""Mock for queries_dir: mock / 'filename' -> file_lookup_func('filename')"""
class QueriesDirMock:
def __truediv__(self, filename):
return file_lookup_func(filename)
return QueriesDirMock()
# Ansible mocking required for importing the module (it imports from ansible.plugins.callback.CallbackBase)
class MockCallbackBase:
def __init__(self):
self._display = mock.MagicMock()
def v2_playbook_on_stats(self, stats):
pass
_mock_callback_module = mock.MagicMock()
_mock_callback_module.CallbackBase = MockCallbackBase
@pytest.fixture(autouse=True)
def _mock_ansible_modules():
"""Temporarily inject fake ansible modules so the callback plugin can be imported."""
with mock.patch.dict(
sys.modules,
{
'ansible': mock.MagicMock(),
'ansible.plugins': mock.MagicMock(),
'ansible.plugins.callback': _mock_callback_module,
'ansible.cli': mock.MagicMock(),
'ansible.cli.galaxy': mock.MagicMock(),
'ansible.release': mock.MagicMock(__version__='2.16.0'),
'ansible.galaxy': mock.MagicMock(),
'ansible.galaxy.collection': mock.MagicMock(),
'ansible.utils': mock.MagicMock(),
'ansible.utils.collection_loader': mock.MagicMock(),
'ansible.constants': mock.MagicMock(),
},
):
yield
class TestListExternalQueries:
"""Tests for list_external_queries function."""
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
def test_returns_empty_when_collection_not_installed(self, mock_files):
from awx.playbooks.library.indirect_instance_count import list_external_queries
mock_files.side_effect = ModuleNotFoundError("No module named 'ansible_collections.redhat'")
result = list_external_queries('demo', 'external')
assert result == []
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
def test_parses_version_from_filenames(self, mock_files):
from awx.playbooks.library.indirect_instance_count import list_external_queries
mock_file_1 = mock.Mock()
mock_file_1.name = 'demo.external.1.0.0.yml'
mock_file_2 = mock.Mock()
mock_file_2.name = 'demo.external.2.1.0.yml'
mock_file_other = mock.Mock()
mock_file_other.name = 'other.collection.1.0.0.yml'
mock_queries_dir = mock.Mock()
mock_queries_dir.iterdir.return_value = [mock_file_1, mock_file_2, mock_file_other]
mock_files.return_value = create_chainable_path_mock(mock_queries_dir)
result = list_external_queries('demo', 'external')
assert len(result) == 2
assert Version('1.0.0') in result
assert Version('2.1.0') in result
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
def test_skips_invalid_versions(self, mock_files):
from awx.playbooks.library.indirect_instance_count import list_external_queries
mock_file_valid = mock.Mock()
mock_file_valid.name = 'demo.external.1.0.0.yml'
mock_file_invalid = mock.Mock()
mock_file_invalid.name = 'demo.external.invalid.yml'
mock_queries_dir = mock.Mock()
mock_queries_dir.iterdir.return_value = [mock_file_valid, mock_file_invalid]
mock_files.return_value = create_chainable_path_mock(mock_queries_dir)
result = list_external_queries('demo', 'external')
assert len(result) == 1
assert Version('1.0.0') in result
class TestVersionFallback:
"""Tests for version fallback logic (AC7.4-AC7.9)."""
@mock.patch('awx.playbooks.library.indirect_instance_count._get_query_file_dir')
def test_exact_match_preferred(self, mock_get_dir):
"""AC7.4: Exact version match is preferred over fallback version."""
from awx.playbooks.library.indirect_instance_count import find_external_query_with_fallback
mock_exact_file = mock.Mock()
mock_exact_file.exists.return_value = True
mock_exact_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('exact_version_query'))
mock_exact_file.open.return_value.__exit__ = mock.Mock(return_value=False)
mock_get_dir.return_value = create_queries_dir_mock(lambda f: mock_exact_file)
content, fallback_used, version = find_external_query_with_fallback('demo', 'external', '2.5.0')
assert content == 'exact_version_query'
assert fallback_used is False
assert version == '2.5.0'
@mock.patch('awx.playbooks.library.indirect_instance_count.list_external_queries')
@mock.patch('awx.playbooks.library.indirect_instance_count._get_query_file_dir')
def test_fallback_nearest_lower_same_major(self, mock_get_dir, mock_list):
"""AC7.5: Fallback selects nearest lower version within same major version.
When installed is 4.5.0 and 4.0.0/4.1.0 are available, selects 4.1.0.
"""
from awx.playbooks.library.indirect_instance_count import find_external_query_with_fallback
mock_list.return_value = [Version('4.0.0'), Version('4.1.0')]
mock_exact_file = mock.Mock(exists=mock.Mock(return_value=False))
mock_fallback_file = mock.Mock()
mock_fallback_file.exists.return_value = True
mock_fallback_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('fallback_query'))
mock_fallback_file.open.return_value.__exit__ = mock.Mock(return_value=False)
def file_lookup(filename):
return mock_fallback_file if '4.1.0' in filename else mock_exact_file
mock_get_dir.return_value = create_queries_dir_mock(file_lookup)
content, fallback_used, version = find_external_query_with_fallback('community', 'vmware', '4.5.0')
assert content == 'fallback_query'
assert fallback_used is True
assert version == '4.1.0'
@mock.patch('awx.playbooks.library.indirect_instance_count.list_external_queries')
@mock.patch('awx.playbooks.library.indirect_instance_count._get_query_file_dir')
def test_fallback_respects_major_version_boundary(self, mock_get_dir, mock_list):
"""Test that fallback does NOT cross major version boundaries.
When installed version is 6.0.0 and only 5.0.0 query exists,
no fallback should occur because major versions differ.
"""
from awx.playbooks.library.indirect_instance_count import find_external_query_with_fallback
mock_list.return_value = [Version('5.0.0')]
# Mock exact file (6.0.0) to not exist
mock_exact_file = mock.Mock(exists=mock.Mock(return_value=False))
# Mock fallback file (5.0.0) to exist - if major version check is broken,
# this file would be incorrectly selected
mock_fallback_file = mock.Mock()
mock_fallback_file.exists.return_value = True
mock_fallback_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('wrong_major_version_query'))
mock_fallback_file.open.return_value.__exit__ = mock.Mock(return_value=False)
def file_lookup(filename):
return mock_fallback_file if '5.0.0' in filename else mock_exact_file
mock_get_dir.return_value = create_queries_dir_mock(file_lookup)
content, fallback_used, version = find_external_query_with_fallback('community', 'vmware', '6.0.0')
# Should NOT fall back to 5.0.0 because major version differs (5 vs 6)
assert content is None
assert fallback_used is False
@mock.patch('awx.playbooks.library.indirect_instance_count.list_external_queries')
@mock.patch('awx.playbooks.library.indirect_instance_count._get_query_file_dir')
def test_no_fallback_when_incompatible(self, mock_get_dir, mock_list):
"""AC7.7: No fallback when all available versions are higher than installed.
When installed version is 3.8.0 and only 4.0.0 and 5.0.0 exist,
no fallback should occur because both are higher than installed.
"""
from awx.playbooks.library.indirect_instance_count import find_external_query_with_fallback
mock_list.return_value = [Version('4.0.0'), Version('5.0.0')]
# Mock exact file (3.8.0) to not exist
mock_exact_file = mock.Mock(exists=mock.Mock(return_value=False))
# Mock available files to exist - if version filtering is broken,
# one of these would be incorrectly selected
mock_available_file = mock.Mock()
mock_available_file.exists.return_value = True
mock_available_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('higher_version_query'))
mock_available_file.open.return_value.__exit__ = mock.Mock(return_value=False)
def file_lookup(filename):
if '4.0.0' in filename or '5.0.0' in filename:
return mock_available_file
return mock_exact_file
mock_get_dir.return_value = create_queries_dir_mock(file_lookup)
content, fallback_used, version = find_external_query_with_fallback('community', 'vmware', '3.8.0')
# Should NOT fall back to 4.0.0 or 5.0.0 because both are higher than 3.8.0
assert content is None
assert fallback_used is False
@mock.patch('awx.playbooks.library.indirect_instance_count.list_external_queries')
@mock.patch('awx.playbooks.library.indirect_instance_count._get_query_file_dir')
def test_fallback_selection_logic(self, mock_get_dir, mock_list):
"""AC7.9: Complex fallback scenario with multiple candidates.
When installed is 4.5.0 and 4.0.0, 4.1.0, 5.0.0 are available,
selects 4.1.0 (highest compatible within same major, <= installed).
"""
from awx.playbooks.library.indirect_instance_count import find_external_query_with_fallback
mock_list.return_value = [Version('4.0.0'), Version('4.1.0'), Version('5.0.0')]
mock_exact_file = mock.Mock(exists=mock.Mock(return_value=False))
mock_fallback_file = mock.Mock()
mock_fallback_file.exists.return_value = True
mock_fallback_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('query_4.1.0'))
mock_fallback_file.open.return_value.__exit__ = mock.Mock(return_value=False)
def file_lookup(filename):
return mock_fallback_file if '4.1.0' in filename else mock_exact_file
mock_get_dir.return_value = create_queries_dir_mock(file_lookup)
content, fallback_used, version = find_external_query_with_fallback('community', 'vmware', '4.5.0')
assert version == '4.1.0'
assert fallback_used is True
assert content == 'query_4.1.0'
class TestExternalQueryDiscovery:
"""Tests for callback plugin query discovery (AC7.1-AC7.3)."""
@mock.patch('awx.playbooks.library.indirect_instance_count.list_collections')
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
@mock.patch('awx.playbooks.library.indirect_instance_count.find_external_query_with_fallback')
@mock.patch.dict('os.environ', {'AWX_ISOLATED_DATA_DIR': '/tmp/artifacts'})
def test_precedence_embedded_over_external(self, mock_fallback, mock_files, mock_list_collections):
"""AC7.1: Embedded query takes precedence when both embedded and external exist."""
from awx.playbooks.library.indirect_instance_count import CallbackModule
mock_list_collections.return_value = [mock.Mock(namespace='demo', name='query', ver='1.0.0', fqcn='demo.query')]
mock_embedded_file = mock.Mock()
mock_embedded_file.exists.return_value = True
mock_embedded_file.open.return_value.__enter__ = mock.Mock(return_value=StringIO('embedded_query'))
mock_embedded_file.open.return_value.__exit__ = mock.Mock(return_value=False)
mock_files.return_value = create_chainable_path_mock(mock_embedded_file)
callback = CallbackModule()
callback._display = mock.Mock()
with mock.patch('builtins.open', mock.mock_open()):
with mock.patch('json.dumps', return_value='{}'):
callback.v2_playbook_on_stats(mock.Mock())
mock_fallback.assert_not_called()
callback._display.vv.assert_called()
@mock.patch('awx.playbooks.library.indirect_instance_count.list_collections')
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
@mock.patch('awx.playbooks.library.indirect_instance_count.find_external_query_with_fallback')
@mock.patch.dict('os.environ', {'AWX_ISOLATED_DATA_DIR': '/tmp/artifacts'})
def test_external_query_when_embedded_missing(self, mock_fallback, mock_files, mock_list_collections):
"""AC7.2: External query is discovered when embedded query is missing."""
from awx.playbooks.library.indirect_instance_count import CallbackModule
mock_candidate = mock.Mock()
mock_candidate.namespace = 'demo'
mock_candidate.name = 'external'
mock_candidate.ver = '2.5.0'
mock_candidate.fqcn = 'demo.external'
mock_list_collections.return_value = [mock_candidate]
mock_embedded_file = mock.Mock(exists=mock.Mock(return_value=False))
mock_files.return_value = create_chainable_path_mock(mock_embedded_file)
mock_fallback.return_value = ('external_query_content', False, '2.5.0')
callback = CallbackModule()
callback._display = mock.Mock()
with mock.patch('builtins.open', mock.mock_open()):
with mock.patch('json.dumps', return_value='{}'):
callback.v2_playbook_on_stats(mock.Mock())
mock_fallback.assert_called_once_with('demo', 'external', '2.5.0')
callback._display.v.assert_called()
@mock.patch('awx.playbooks.library.indirect_instance_count.list_collections')
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
@mock.patch('awx.playbooks.library.indirect_instance_count.find_external_query_with_fallback')
@mock.patch.dict('os.environ', {'AWX_ISOLATED_DATA_DIR': '/tmp/artifacts'})
def test_no_query_when_both_missing(self, mock_fallback, mock_files, mock_list_collections):
"""AC7.3: No query is used when both embedded and external queries are missing."""
from awx.playbooks.library.indirect_instance_count import CallbackModule
mock_list_collections.return_value = [mock.Mock(namespace='unknown', name='collection', ver='1.0.0', fqcn='unknown.collection')]
mock_embedded_file = mock.Mock(exists=mock.Mock(return_value=False))
mock_files.return_value = create_chainable_path_mock(mock_embedded_file)
mock_fallback.return_value = (None, False, None)
callback = CallbackModule()
callback._display = mock.Mock()
with mock.patch('builtins.open', mock.mock_open()):
with mock.patch('json.dumps', return_value='{}'):
callback.v2_playbook_on_stats(mock.Mock())
mock_fallback.assert_called_once()
@mock.patch('awx.playbooks.library.indirect_instance_count.list_collections')
@mock.patch('awx.playbooks.library.indirect_instance_count.files')
@mock.patch('awx.playbooks.library.indirect_instance_count.find_external_query_with_fallback')
@mock.patch.dict('os.environ', {'AWX_ISOLATED_DATA_DIR': '/tmp/artifacts'})
def test_info_log_on_fallback(self, mock_fallback, mock_files, mock_list_collections):
"""AC7.8: Log message is emitted when fallback version is used.
Verifies that when a fallback version is used, a log message is emitted
containing both the fallback version and the collection FQCN.
Note: AC7.8 specifies 'warning logs' but implementation uses verbose/info
level (_display.v) as this is informational rather than a warning condition.
"""
from awx.playbooks.library.indirect_instance_count import CallbackModule
mock_list_collections.return_value = [mock.Mock(namespace='community', name='vmware', ver='4.5.0', fqcn='community.vmware')]
mock_embedded_file = mock.Mock(exists=mock.Mock(return_value=False))
mock_files.return_value = create_chainable_path_mock(mock_embedded_file)
mock_fallback.return_value = ('fallback_query_content', True, '4.1.0')
callback = CallbackModule()
callback._display = mock.Mock()
with mock.patch('builtins.open', mock.mock_open()):
with mock.patch('json.dumps', return_value='{}'):
callback.v2_playbook_on_stats(mock.Mock())
callback._display.v.assert_called()
call_args = callback._display.v.call_args[0][0]
assert '4.1.0' in call_args
assert 'community.vmware' in call_args
class TestPrivateDataDirIntegration:
"""Tests for vendor collection copying (AC7.10-AC7.11)."""
@mock.patch('awx.main.tasks.jobs.flag_enabled')
@mock.patch('awx.main.tasks.jobs.shutil.copytree')
@mock.patch('awx.main.tasks.jobs.os.path.exists')
def test_vendor_collections_copied(self, mock_exists, mock_copytree, mock_flag):
"""AC7.10: build_private_data_files() copies vendor collections to private_data_dir."""
from awx.main.tasks.jobs import BaseTask
mock_flag.return_value = True
mock_exists.return_value = True
task = BaseTask()
task.instance = mock.Mock()
task.cleanup_paths = []
task.build_private_data = mock.Mock(return_value=None)
private_data_dir = '/tmp/awx_123_abc'
task.build_private_data_files(task.instance, private_data_dir)
mock_copytree.assert_called_once_with('/var/lib/awx/vendor_collections', f'{private_data_dir}/vendor_collections')
@mock.patch('awx.main.tasks.jobs.flag_enabled')
@mock.patch('awx.main.tasks.jobs.logger')
@mock.patch('awx.main.tasks.jobs.shutil.copytree')
@mock.patch('awx.main.tasks.jobs.os.path.exists')
def test_missing_source_handled_gracefully(self, mock_exists, mock_copytree, mock_logger, mock_flag):
"""AC7.11: Collection copy handles missing source directory gracefully."""
from awx.main.tasks.jobs import BaseTask
mock_flag.return_value = True
mock_exists.return_value = False
task = BaseTask()
task.instance = mock.Mock()
task.cleanup_paths = []
task.build_private_data = mock.Mock(return_value=None)
private_data_dir = '/tmp/awx_123_abc'
result = task.build_private_data_files(task.instance, private_data_dir)
# copytree should not be called when source doesn't exist
mock_copytree.assert_not_called()
# Function should complete without raising an exception
assert result is not None

View File

@@ -1,6 +1,8 @@
# -*- coding: utf-8 -*-
import json
import os
import shutil
import tempfile
from pathlib import Path
import fcntl
@@ -58,12 +60,14 @@ class TestJobExecution(object):
@pytest.fixture
def private_data_dir(tmp_path):
private_data = tmp_path / 'awx_pdd'
private_data.mkdir()
def private_data_dir():
private_data = tempfile.mkdtemp(prefix='awx_')
for subfolder in ('inventory', 'env'):
(private_data / subfolder).mkdir()
return str(private_data)
runner_subfolder = os.path.join(private_data, subfolder)
if not os.path.exists(runner_subfolder):
os.mkdir(runner_subfolder)
yield private_data
shutil.rmtree(private_data, True)
@pytest.fixture
@@ -552,8 +556,7 @@ class TestGenericRun:
task._write_extra_vars_file = mock.Mock()
with mock.patch('awx.main.tasks.jobs.settings.AWX_TASK_ENV', {'FOO': 'BAR'}):
with mock.patch.object(task, 'build_credentials_list', return_value=[], autospec=True):
env = task.build_env(job, private_data_dir)
env = task.build_env(job, private_data_dir)
assert env['FOO'] == 'BAR'
@@ -621,11 +624,6 @@ class TestAdhocRun(TestJobExecution):
class TestJobCredentials(TestJobExecution):
@pytest.fixture(autouse=True)
def mock_flag_enabled(self):
with mock.patch('awx.main.tasks.jobs.flag_enabled', return_value=False):
yield
@pytest.fixture
def job(self, execution_environment):
job = Job(pk=1, inventory=Inventory(pk=1), project=Project(pk=1))
@@ -651,9 +649,7 @@ class TestJobCredentials(TestJobExecution):
)
with mock.patch.object(UnifiedJob, 'credentials', credentials_mock):
# Mock build_credentials_list to work with the cached credentials mechanism
with mock.patch.object(jobs.RunJob, 'build_credentials_list', return_value=job._credentials, autospec=True):
yield job
yield job
@pytest.fixture
def update_model_wrapper(self, job):
@@ -1159,11 +1155,6 @@ class TestProjectUpdateRefspec(TestJobExecution):
class TestInventoryUpdateCredentials(TestJobExecution):
@pytest.fixture(autouse=True)
def mock_flag_enabled(self):
with mock.patch('awx.main.tasks.jobs.flag_enabled', return_value=False):
yield
@pytest.fixture
def inventory_update(self, execution_environment):
return InventoryUpdate(pk=1, execution_environment=execution_environment, inventory_source=InventorySource(pk=1, inventory=Inventory(pk=1)))
@@ -1583,7 +1574,7 @@ def test_managed_injector_redaction(injector_cls):
assert 'very_secret_value' not in str(build_safe_env(env))
def test_job_run_no_ee(mock_me, mock_create_partition, private_data_dir):
def test_job_run_no_ee(mock_me, mock_create_partition):
org = Organization(pk=1)
proj = Project(pk=1, organization=org)
job = Job(project=proj, organization=org, inventory=Inventory(pk=1))

View File

@@ -330,13 +330,17 @@ class TestHostnameRegexValidator:
def test_bad_call(self, regex_expr, re_flags):
h = HostnameRegexValidator(regex=regex_expr, flags=re_flags)
with pytest.raises(ValidationError, match=r"^\['illegal characters detected in hostname=@#\$%\)\$#\(TUFAS_DG. Please verify.'\]$"):
try:
h("@#$%)$#(TUFAS_DG")
except ValidationError as e:
assert e.message is not None
def test_good_call_with_inverse(self, regex_expr, re_flags, inverse_match=True):
h = HostnameRegexValidator(regex=regex_expr, flags=re_flags, inverse_match=inverse_match)
with pytest.raises(ValidationError, match=r"^\['Enter a valid value.'\]$"):
try:
h("1.2.3.4")
except ValidationError as e:
assert e.message is not None
def test_bad_call_with_inverse(self, regex_expr, re_flags, inverse_match=True):
h = HostnameRegexValidator(regex=regex_expr, flags=re_flags, inverse_match=inverse_match)

View File

@@ -1,4 +1,6 @@
import shutil
import os
from uuid import uuid4
import pytest
@@ -22,11 +24,17 @@ def test_switch_paths(container_path, host_path):
assert get_incontainer_path(host_path, private_data_dir) == container_path
def test_symlink_isolation_dir(tmp_path):
src_path = tmp_path / 'symlink_src'
dst_path = tmp_path / 'symlink_dst'
def test_symlink_isolation_dir(request):
rand_str = str(uuid4())[:8]
dst_path = f'/tmp/ee_{rand_str}_symlink_dst'
src_path = f'/tmp/ee_{rand_str}_symlink_src'
src_path.mkdir()
def remove_folders():
os.unlink(dst_path)
shutil.rmtree(src_path)
request.addfinalizer(remove_folders)
os.mkdir(src_path)
os.symlink(src_path, dst_path)
pdd = f'{dst_path}/awx_xxx'

View File

@@ -0,0 +1,207 @@
# Copyright (c) 2024 Ansible, Inc.
# All Rights Reserved.
from unittest import mock
from awx.main.utils.proxy import get_first_remote_host_from_headers, is_proxy_in_headers
class TestGetFirstRemoteHostFromHeaders:
"""Tests for get_first_remote_host_from_headers function."""
def _make_mock_request(self, environ):
"""Create a mock request with the given environ dict."""
request = mock.MagicMock()
request.environ = environ
return request
def test_single_value_headers(self):
"""Test extraction from headers with single values (no commas)."""
request = self._make_mock_request(
{
"REMOTE_ADDR": "192.168.1.1",
"REMOTE_HOST": "client.example.com",
}
)
headers = ["REMOTE_ADDR", "REMOTE_HOST"]
result = get_first_remote_host_from_headers(request, headers)
assert result == {"192.168.1.1", "client.example.com"}
def test_comma_separated_only_first_entry(self):
"""Test that only the first entry is extracted from comma-separated values."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": "10.0.0.1, 192.168.1.1, 172.16.0.1",
}
)
headers = ["HTTP_X_FORWARDED_FOR"]
result = get_first_remote_host_from_headers(request, headers)
# Only the first IP should be included
assert result == {"10.0.0.1"}
# Subsequent IPs should NOT be included
assert "192.168.1.1" not in result
assert "172.16.0.1" not in result
def test_comma_separated_with_whitespace(self):
"""Test that whitespace is properly stripped from first entry."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": " 10.0.0.1 , 192.168.1.1",
}
)
headers = ["HTTP_X_FORWARDED_FOR"]
result = get_first_remote_host_from_headers(request, headers)
assert result == {"10.0.0.1"}
def test_multiple_headers_with_comma_separated(self):
"""Test multiple headers where some have comma-separated values."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": "client.example.com, proxy1.example.com, proxy2.example.com",
"REMOTE_ADDR": "172.16.0.1",
"REMOTE_HOST": "proxy2.example.com",
}
)
headers = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR", "REMOTE_HOST"]
result = get_first_remote_host_from_headers(request, headers)
# Should have first entry from X-Forwarded-For plus the single values from other headers
assert result == {"client.example.com", "172.16.0.1", "proxy2.example.com"}
# Should NOT have subsequent entries from X-Forwarded-For
assert "proxy1.example.com" not in result
def test_empty_header_value(self):
"""Test handling of empty header values."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": "",
"REMOTE_ADDR": "192.168.1.1",
}
)
headers = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR"]
result = get_first_remote_host_from_headers(request, headers)
assert result == {"192.168.1.1"}
def test_missing_header(self):
"""Test handling of headers that don't exist in environ."""
request = self._make_mock_request(
{
"REMOTE_ADDR": "192.168.1.1",
}
)
headers = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR", "REMOTE_HOST"]
result = get_first_remote_host_from_headers(request, headers)
assert result == {"192.168.1.1"}
def test_empty_headers_list(self):
"""Test with no headers specified."""
request = self._make_mock_request(
{
"REMOTE_ADDR": "192.168.1.1",
}
)
headers = []
result = get_first_remote_host_from_headers(request, headers)
assert result == set()
def test_whitespace_only_first_entry(self):
"""Test handling when first entry is whitespace only."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": " , 192.168.1.1",
}
)
headers = ["HTTP_X_FORWARDED_FOR"]
result = get_first_remote_host_from_headers(request, headers)
# Empty/whitespace first entry should be skipped
assert result == set()
def test_single_entry_with_trailing_comma(self):
"""Test single entry that happens to have a trailing comma."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": "10.0.0.1,",
}
)
headers = ["HTTP_X_FORWARDED_FOR"]
result = get_first_remote_host_from_headers(request, headers)
assert result == {"10.0.0.1"}
class TestIsProxyInHeaders:
"""Tests for is_proxy_in_headers function."""
def _make_mock_request(self, environ):
"""Create a mock request with the given environ dict."""
request = mock.MagicMock()
request.environ = environ
return request
def test_proxy_found_in_single_value(self):
"""Test proxy detection in single-value header."""
request = self._make_mock_request(
{
"REMOTE_ADDR": "192.168.1.1",
}
)
result = is_proxy_in_headers(request, ["192.168.1.1"], ["REMOTE_ADDR"])
assert result is True
def test_proxy_found_in_comma_separated(self):
"""Test proxy detection in comma-separated header value."""
request = self._make_mock_request(
{
"HTTP_X_FORWARDED_FOR": "10.0.0.1, 192.168.1.1, 172.16.0.1",
}
)
result = is_proxy_in_headers(request, ["192.168.1.1"], ["HTTP_X_FORWARDED_FOR"])
assert result is True
def test_proxy_not_found(self):
"""Test when proxy is not in any header."""
request = self._make_mock_request(
{
"REMOTE_ADDR": "10.0.0.1",
}
)
result = is_proxy_in_headers(request, ["192.168.1.1"], ["REMOTE_ADDR"])
assert result is False
def test_multiple_proxies_one_match(self):
"""Test with multiple allowed proxies, one matches."""
request = self._make_mock_request(
{
"REMOTE_HOST": "proxy.example.com",
}
)
result = is_proxy_in_headers(
request,
["proxy1.example.com", "proxy.example.com", "proxy2.example.com"],
["REMOTE_HOST"],
)
assert result is True

View File

@@ -48,16 +48,15 @@ def could_be_playbook(project_path, dir_path, filename):
# show up.
matched = False
try:
with codecs.open(playbook_path, 'r', encoding='utf-8', errors='ignore') as f:
for n, line in enumerate(f):
if valid_playbook_re.match(line):
matched = True
break
# Any YAML file can also be encrypted with vault;
# allow these to be used as the main playbook.
elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
matched = True
break
for n, line in enumerate(codecs.open(playbook_path, 'r', encoding='utf-8', errors='ignore')):
if valid_playbook_re.match(line):
matched = True
break
# Any YAML file can also be encrypted with vault;
# allow these to be used as the main playbook.
elif n == 0 and line.startswith('$ANSIBLE_VAULT;'):
matched = True
break
except IOError:
return None
if not matched:

View File

@@ -55,8 +55,6 @@ def construct_rsyslog_conf_template(settings=settings):
)
def escape_quotes(x):
if x is None:
return ''
return x.replace('"', '\\"')
if not enabled:

View File

@@ -45,3 +45,38 @@ def delete_headers_starting_with_http(request: Request, headers: list[str]):
for header in headers:
if header.startswith('HTTP_'):
request.environ.pop(header, None)
def get_first_remote_host_from_headers(request: Request, headers: list[str]) -> set[str]:
"""
Extract remote host addresses from headers, considering only the first entry
in comma-separated values.
For headers like X-Forwarded-For that may contain multiple IPs (e.g., "client, proxy1, proxy2"),
only the first entry (the original client) is considered.
Example:
request.environ = {
"HTTP_X_FORWARDED_FOR": "10.0.0.1, 192.168.1.1, 172.16.0.1",
"REMOTE_ADDR": "192.168.1.1",
"REMOTE_HOST": "proxy.example.com"
}
headers = ["HTTP_X_FORWARDED_FOR", "REMOTE_ADDR", "REMOTE_HOST"]
Returns: {"10.0.0.1", "192.168.1.1", "proxy.example.com"}
(Only the first IP "10.0.0.1" from X-Forwarded-For, not the full chain)
request: The DRF/Django request. request.environ dict will be used for extracting hosts
headers: A list of header keys to check for remote host values
"""
remote_hosts = set()
for header in headers:
header_value = request.environ.get(header, '')
if header_value:
# Only take the first entry if comma-separated
first_value = header_value.split(',')[0].strip()
if first_value:
remote_hosts.add(first_value)
return remote_hosts

View File

@@ -139,7 +139,7 @@ class WebsocketRelayConnection:
except json.JSONDecodeError:
logmsg = "Failed to decode message from web node"
if logger.isEnabledFor(logging.DEBUG):
logmsg = "{} {}".format(logmsg, msg.data)
logmsg = "{} {}".format(logmsg, payload)
logger.warning(logmsg)
continue
@@ -242,7 +242,7 @@ class WebSocketRelayManager(object):
except json.JSONDecodeError:
logmsg = "Failed to decode message from pg_notify channel `web_ws_heartbeat`"
if logger.isEnabledFor(logging.DEBUG):
logmsg = "{} {}".format(logmsg, notif.payload)
logmsg = "{} {}".format(logmsg, payload)
logger.warning(logmsg)
continue

View File

@@ -21,11 +21,8 @@ DOCUMENTATION = '''
import os
import json
import re
from importlib.resources import files
from packaging.version import Version, InvalidVersion
from ansible.plugins.callback import CallbackBase
# NOTE: in Ansible 1.2 or later general logging is available without
@@ -44,101 +41,6 @@ from ansible.galaxy.collection import find_existing_collections
from ansible.utils.collection_loader import AnsibleCollectionConfig
import ansible.constants as C
# External query path constants
EXTERNAL_QUERY_COLLECTION = 'ansible_collections.redhat.indirect_accounting'
def _get_query_file_dir():
"""Return the query file directory or None."""
try:
queries_dir = files(EXTERNAL_QUERY_COLLECTION) / 'extensions' / 'audit' / 'external_queries'
except ModuleNotFoundError:
return None
if not queries_dir.is_dir():
return None
return queries_dir
def list_external_queries(namespace, name):
"""List all available external query versions for a collection.
Args:
namespace: Collection namespace (e.g., 'community')
name: Collection name (e.g., 'vmware')
Returns:
List of Version objects for all available query files
matching the namespace.name pattern.
"""
versions = []
if not (queries_dir := _get_query_file_dir()):
return versions
# Pattern: namespace.name.X.Y.Z.yml where X.Y.Z is the version
pattern = re.compile(rf'^{re.escape(namespace)}\.{re.escape(name)}\.(.+)\.yml$')
for query_file in queries_dir.iterdir():
match = pattern.match(query_file.name)
if match:
version_str = match.group(1)
try:
versions.append(Version(version_str))
except InvalidVersion:
# Skip files with invalid version strings
pass
return versions
def find_external_query_with_fallback(namespace, name, installed_version):
"""Find external query file with semantic version fallback.
Args:
namespace: Collection namespace (e.g., 'community')
name: Collection name (e.g., 'vmware')
installed_version: Version string of installed collection (e.g., '4.5.0')
Returns:
Tuple of (query_content, fallback_used, fallback_version) or (None, False, None)
- query_content: The query file content if found
- fallback_used: True if a fallback version was used instead of exact match
- fallback_version: The version string used (for logging)
"""
if not (queries_dir := _get_query_file_dir()):
return None, False, None
# 1. Try exact version match first
exact_file = queries_dir / f'{namespace}.{name}.{installed_version}.yml'
if exact_file.exists():
with exact_file.open('r') as f:
return f.read(), False, installed_version
# 2. Find compatible fallback (same major version, nearest lower version)
try:
installed_version_object = Version(installed_version)
except InvalidVersion:
# Can't do version comparison for fallback
return None, False, None
available_versions = list_external_queries(namespace, name)
if not available_versions:
return None, False, None
# Filter to same major version and versions <= installed version
compatible_versions = [v for v in available_versions if v.major == installed_version_object.major and v <= installed_version_object]
if not compatible_versions:
return None, False, None
# Select nearest lower version - highest compatible version
fallback_version_object = max(compatible_versions)
fallback_version_str = str(fallback_version_object)
fallback_file = queries_dir / f'{namespace}.{name}.{fallback_version_str}.yml'
if fallback_file.exists():
with fallback_file.open('r') as f:
return f.read(), True, fallback_version_str
return None, False, None
@with_collection_artifacts_manager
def list_collections(artifacts_manager=None):
@@ -175,22 +77,10 @@ class CallbackModule(CallbackBase):
'version': candidate.ver,
}
# 1. Check for embedded query file (takes precedence)
embedded_query_file = files(f'ansible_collections.{candidate.namespace}.{candidate.name}') / 'extensions' / 'audit' / 'event_query.yml'
if embedded_query_file.exists():
with embedded_query_file.open('r') as f:
query_file = files(f'ansible_collections.{candidate.namespace}.{candidate.name}') / 'extensions' / 'audit' / 'event_query.yml'
if query_file.exists():
with query_file.open('r') as f:
collection_print['host_query'] = f.read()
self._display.vv(f"Using embedded query for {candidate.fqcn} v{candidate.ver}")
else:
# 2. Check for external query file with version fallback
query_content, fallback_used, version_used = find_external_query_with_fallback(candidate.namespace, candidate.name, candidate.ver)
if query_content:
collection_print['host_query'] = query_content
if fallback_used:
# AC5.6: Log when fallback is used
self._display.v(f"Using external query {version_used} for {candidate.fqcn} v{candidate.ver}.")
else:
self._display.v(f"Using external query for {candidate.fqcn} v{candidate.ver}")
collections_print[candidate.fqcn] = collection_print

View File

@@ -236,7 +236,7 @@
changed_when: "'was installed successfully' in galaxy_result.stdout"
when:
- roles_enabled | bool
- req_file | length > 0
- req_file
tags:
- install_roles
@@ -255,7 +255,7 @@
when:
- "ansible_version.full is version_compare('2.9', '>=')"
- collections_enabled | bool
- req_file | length > 0
- req_file
tags:
- install_collections
@@ -276,7 +276,7 @@
- "ansible_version.full is version_compare('2.10', '>=')"
- collections_enabled | bool
- roles_enabled | bool
- req_file | length > 0
- req_file
tags:
- install_collections
- install_roles

View File

@@ -63,15 +63,6 @@ assert_production_settings(DYNACONF, settings_dir, settings_file_path)
# Load envvars at the end to allow them to override everything loaded so far
load_envvars(DYNACONF)
# When deployed as part of AAP (RESOURCE_SERVER__URL is set), enforce JWT-only
# authentication. This ensures all requests go through the gateway and prevents
# direct API access to Controller bypassing the platform's authentication.
if DYNACONF.get('RESOURCE_SERVER__URL', None):
DYNACONF.set(
"REST_FRAMEWORK__DEFAULT_AUTHENTICATION_CLASSES",
['ansible_base.jwt_consumer.awx.auth.AwxJWTAuthentication'],
)
# This must run after all custom settings are loaded
DYNACONF.update(
merge_application_name(DYNACONF),

View File

@@ -774,7 +774,7 @@ LOGGING = {
'awx.conf.settings': {'handlers': ['null'], 'level': 'WARNING'},
'awx.main': {'handlers': ['null']},
'awx.main.commands.run_callback_receiver': {'handlers': ['callback_receiver'], 'level': 'INFO'}, # very noisey debug-level logs
'awx.main.dispatch': {'handlers': ['task_system']},
'awx.main.dispatch': {'handlers': ['dispatcher']},
'awx.main.consumers': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'INFO'},
'awx.main.rsyslog_configurer': {'handlers': ['rsyslog_configurer']},
'awx.main.cache_clear': {'handlers': ['cache_clear']},
@@ -1134,7 +1134,6 @@ OPA_REQUEST_RETRIES = 2 # The number of retry attempts for connecting to the OP
# feature flags
FEATURE_INDIRECT_NODE_COUNTING_ENABLED = False
FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED = False
# Dispatcher worker lifetime. If set to None, workers will never be retired
# based on age. Note workers will finish their last task before retiring if

View File

@@ -19,9 +19,6 @@ SECRET_KEY = None
# See https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
ALLOWED_HOSTS = []
# In production, trust the X-Forwarded-For header set by the reverse proxy
REMOTE_HOST_HEADERS = ['HTTP_X_FORWARDED_FOR']
# Ansible base virtualenv paths and enablement
# only used for deprecated fields and management commands for them
BASE_VENV_PATH = os.path.realpath("/var/lib/awx/venv")

View File

@@ -1,8 +1,8 @@
# This is a cross-platform list tracking distribution packages needed by tests;
# see https://docs.openstack.org/infra/bindep/ for additional information.
python3-pytz [platform:centos-8 platform:rhel-8 platform:centos-9 platform:rhel-9]
python38-pytz [platform:centos-8 platform:rhel-8]
# awxkit
python3-requests [platform:centos-8 platform:rhel-8 platform:centos-9 platform:rhel-9]
python3-pyyaml [platform:centos-8 platform:rhel-8 platform:centos-9 platform:rhel-9]
python38-requests [platform:centos-8 platform:rhel-8]
python38-pyyaml [platform:centos-8 platform:rhel-8]

View File

@@ -12,6 +12,15 @@ class ConnectionException(exc.Common):
pass
class TokenAuth(requests.auth.AuthBase):
def __init__(self, token):
self.token = token
def __call__(self, request):
request.headers['Authorization'] = 'Bearer {0.token}'.format(self)
return request
def log_elapsed(r, *args, **kwargs): # requests hook to display API elapsed time
log.debug('"{0.request.method} {0.url}" elapsed: {0.elapsed}'.format(r))
@@ -37,7 +46,7 @@ class Connection(object):
self.get(config.api_base_path) # this causes a cookie w/ the CSRF token to be set
return dict(next=next)
def login(self, username=None, password=None, **kwargs):
def login(self, username=None, password=None, token=None, **kwargs):
if username and password:
_next = kwargs.get('next')
if _next:
@@ -49,14 +58,11 @@ class Connection(object):
self.session_cookie_name = historical_response.headers.get('X-API-Session-Cookie-Name')
self.session_id = self.session.cookies.get(self.session_cookie_name, None)
if self.session_id is None and config.get("api_base_path") == "/api/controller/":
# Use gateway session cookie name if controller session cookie name is not found
self.session_cookie_name = "gateway_sessionid"
self.session_id = self.session.cookies.get(self.session_cookie_name, None)
self.uses_session_cookie = True
else:
self.session.auth = (username, password)
elif token:
self.session.auth = TokenAuth(token)
else:
self.session.auth = None

View File

@@ -61,7 +61,7 @@ def separate_async_optionals(creation_order):
continue
by_count = defaultdict(set)
has_creates = [cand for cand in group if hasattr(cand, 'dependencies')]
counts = dict.fromkeys(has_creates, 0)
counts = {has_create: 0 for has_create in has_creates}
for has_create in has_creates:
for dependency in has_create.dependencies:
for compared in [cand for cand in has_creates if cand != has_create]:
@@ -212,7 +212,7 @@ class HasCreate(object):
dependency_store = kw.get('ds')
if dependency_store is None:
deps = self.dependencies + self.optional_dependencies
self._dependency_store = dict.fromkeys(deps)
self._dependency_store = {base_subclass: None for base_subclass in deps}
self.ds = DSAdapter(self.__class__.__name__, self._dependency_store)
else:
self._dependency_store = dependency_store.dependency_store

View File

@@ -31,23 +31,7 @@ class User(HasCreate, base.Base):
payload = self.create_payload(username=username, password=password, **kwargs)
self.password = payload.password
ctrl_users_api = Users(self.connection)
# Check if API base path is set to controller, then use gateway endpoint
if config.get("api_base_path") == "/api/controller/":
# Use gateway endpoint for user creation
gw_users_api = Users(self.connection)
gw_users_api.endpoint = "/api/gateway/v1/users/"
# Cleanup controller attributes
payload["is_platform_auditor"] = payload.get("is_system_auditor")
payload.pop("is_system_auditor")
# Create gw user
gw_user = gw_users_api.post(payload)
user = ctrl_users_api.get(username=gw_user.username).results.pop()
user.json["password"] = payload.password
self.update_identity(user)
else:
# Use default endpoint
self.update_identity(ctrl_users_api.post(payload))
self.update_identity(Users(self.connection).post(payload))
if organization:
organization.add_user(self)

View File

@@ -80,62 +80,26 @@ class CLI(object):
def help(self):
return '--help' in self.argv or '-h' in self.argv
def _get_non_option_args(self, before_help=False):
"""Extract non-option arguments from argv, optionally only those before help flag."""
if before_help and self.help:
# Find position of help flag
help_pos = next((i for i, arg in enumerate(self.argv) if arg in ('--help', '-h')), len(self.argv))
args_to_check = self.argv[:help_pos]
else:
args_to_check = self.argv
non_option_args = []
i = 0
while i < len(args_to_check):
arg = args_to_check[i]
if arg == 'awx':
# Skip 'awx' token
i += 1
elif arg.startswith('-'):
# This is an option
if '=' in arg:
# Long option with value: --opt=val
i += 1
else:
# Option without embedded value: --opt or -o
i += 1
# Only consume next argument if it exists AND doesn't start with '-'
# This naturally handles flag-only options (like --verbose)
if i < len(args_to_check) and not args_to_check[i].startswith('-'):
i += 1
else:
# This is a positional argument
non_option_args.append(arg)
i += 1
return non_option_args
def _is_main_help_request(self):
"""
Determine if help request is for main CLI (awx --help) vs subcommand (awx users create --help).
Returns True if this is a main CLI help request that should exit early.
"""
if not self.help:
return False
# If there are non-option arguments before help flag, this is subcommand help
return len(self._get_non_option_args(before_help=True)) == 0
def authenticate(self):
"""Configure the current session for authentication.
Uses Basic authentication when AWXKIT_FORCE_BASIC_AUTH environment variable
is set to true, otherwise defaults to session-based authentication.
Authentication priority:
1. Token authentication (if --conf.token provided)
2. Basic authentication (if AWXKIT_FORCE_BASIC_AUTH=true)
3. Session-based authentication (default)
For AAP Gateway environments, set AWXKIT_FORCE_BASIC_AUTH=true to bypass
session login restrictions.
session login restrictions when using username/password.
"""
# Token authentication (if token is provided)
token = self.get_config('token')
if token:
config.use_sessions = False
self.root.connection.login(None, None, token=token)
return
# Check if Basic auth is forced via environment variable
if config.get('force_basic_auth', False):
config.use_sessions = False
@@ -271,16 +235,6 @@ class CLI(object):
subparsers = self.subparsers[self.resource].add_subparsers(dest='action', metavar='action')
subparsers.required = True
# Add manual help handling for resource-level help
# since we disabled add_help=False for resource subparsers
if self.help:
# Check if this is resource-level help (no action specified)
non_option_args = self._get_non_option_args()
if len(non_option_args) == 1 and non_option_args[0] == self.resource:
# Only resource specified, no action - show resource-level help
self.subparsers[self.resource].print_help()
return
# parse the action from OPTIONS
parser = ResourceOptionsParser(self.v2, page, self.resource, subparsers)
if parser.deprecated:
@@ -289,18 +243,6 @@ class CLI(object):
description = colored(description, 'yellow')
self.subparsers[self.resource].description = description
# parse any action arguments FIRST before attempting to parse
if self.resource != 'settings':
for method in ('list', 'modify', 'create'):
if method in parser.parser.choices:
if method == 'list':
http_method = 'GET'
elif method == 'modify' and 'PUT' in parser.options:
http_method = 'PUT'
else:
http_method = 'POST'
parser.build_query_arguments(method, http_method)
if from_sphinx:
# Our Sphinx plugin runs `parse_action` for *every* available
# resource + action in the API so that it can generate usage
@@ -313,6 +255,21 @@ class CLI(object):
self.parser.parse_known_args(self.argv)[0]
except SystemExit:
pass
else:
self.parser.parse_known_args()[0]
# parse any action arguments
if self.resource != 'settings':
for method in ('list', 'modify', 'create'):
if method in parser.parser.choices:
if method == 'list':
http_method = 'GET'
elif method == 'modify' and 'PUT' in parser.options:
http_method = 'PUT'
else:
http_method = 'POST'
parser.build_query_arguments(method, http_method)
if from_sphinx:
parsed, extra = self.parser.parse_known_args(self.argv)
else:
parsed, extra = self.parser.parse_known_args()
@@ -367,7 +324,6 @@ class CLI(object):
self.argv = argv
self.parser = HelpfulArgumentParser(add_help=False)
self.parser.add_argument(
'-h',
'--help',
action='store_true',
help='prints usage information for the awx tool',
@@ -377,13 +333,6 @@ class CLI(object):
add_output_formatting_arguments(self.parser, env)
self.args = self.parser.parse_known_args(self.argv)[0]
# Early return for help to avoid server connection, but only for main CLI help
# Allow subcommand help (like 'awx users create --help') to continue processing
if self.help and self._is_main_help_request():
self.parser.print_help()
sys.exit(0)
self.verbose = self.get_config('verbose')
if self.verbose:
logging.basicConfig(level='DEBUG')

View File

@@ -59,6 +59,12 @@ def add_authentication_arguments(parser, env):
default=env.get('CONTROLLER_PASSWORD', env.get('TOWER_PASSWORD', config_password)),
metavar='TEXT',
)
auth.add_argument(
'--conf.token',
default=env.get('CONTROLLER_OAUTH_TOKEN', env.get('TOWER_OAUTH_TOKEN', None)),
metavar='TEXT',
help='OAuth2 token for authentication (takes precedence over username/password)',
)
auth.add_argument(
'-k',

View File

@@ -126,7 +126,7 @@ class ResourceOptionsParser(object):
if method not in action_map:
continue
method = action_map[method]
parser = self.parser.add_parser(method, help='', add_help=True)
parser = self.parser.add_parser(method, help='')
if method == 'list':
parser.add_argument(
'--all',
@@ -152,7 +152,7 @@ class ResourceOptionsParser(object):
if 'DELETE' in self.allowed_options:
allowed.append('delete')
for method in allowed:
parser = self.parser.add_parser(method, help='', add_help=True)
parser = self.parser.add_parser(method, help='')
self.parser.choices[method].add_argument(
'id', type=functools.partial(pk_or_name, self.v2, self.resource), help='the ID (or unique name) of the resource'
)

View File

@@ -167,9 +167,7 @@ def parse_resource(client, skip_deprecated=False):
if k in DEPRECATED_RESOURCES:
kwargs['aliases'] = [DEPRECATED_RESOURCES[k]]
# Disable automatic help handling for resource subparsers to prevent
# premature help display before action subparsers are built
client.subparsers[k] = subparsers.add_parser(k, help='', add_help=False, **kwargs)
client.subparsers[k] = subparsers.add_parser(k, help='', **kwargs)
resource = client.parser.parse_known_args()[0].resource
if resource in DEPRECATED_RESOURCES.values():

View File

@@ -2,12 +2,9 @@ class Common(Exception):
def __init__(self, status_string='', message=''):
if isinstance(status_string, Exception):
self.status_string = ''
self.msg = message
super().__init__(*status_string.args)
else:
self.status_string = status_string
self.msg = message
super().__init__(status_string, message)
return super(Common, self).__init__(*status_string)
self.status_string = status_string
self.msg = message
def __getitem__(self, val):
return (self.status_string, self.msg)[val]

View File

@@ -44,6 +44,48 @@ def setup_session_auth(cli_args: Optional[List[str]] = None) -> Tuple[CLI, Mock,
return cli, mock_root, mock_load_session
def setup_token_auth(cli_args: Optional[List[str]] = None) -> Tuple[CLI, Mock, Mock]:
"""Set up CLI with mocked connection for Token auth testing"""
cli = CLI()
cli.parse_args(cli_args or ['awx', '--conf.token', 'test-token-abc123'])
mock_root = Mock()
mock_connection = Mock()
mock_root.connection = mock_connection
cli.root = mock_root
return cli, mock_root, mock_connection
def test_token_auth_preserved(monkeypatch):
"""
REGRESSION TEST: Token authentication must still work (existed in 4.6.12)
This test documents the customer's working scenario from 4.6.12:
awx login --conf.host URL --conf.username USER --conf.password PASS
# Returns: {"token": "E*******J"}
awx --conf.host URL --conf.token E*******J job_templates launch ...
# This WORKED in 4.6.12
BREAKING CHANGE: Version 4.6.21 removed token authentication entirely,
causing customer to report: "neither token no username/password are working"
This test will FAIL with current code and PASS once fixed.
"""
cli, mock_root, mock_connection = setup_token_auth(['awx', '--conf.host', 'https://aap-sbx.testbank.com', '--conf.token', 'E1234567890J'])
monkeypatch.setattr(config, 'force_basic_auth', False)
# Execute authentication
cli.authenticate()
# Token auth should call login with token parameter
mock_connection.login.assert_called_once_with(None, None, token='E1234567890J')
# Should NOT use sessions when token is provided
assert not config.use_sessions
def test_basic_auth_enabled(monkeypatch):
"""Test that AWXKIT_FORCE_BASIC_AUTH=true enables Basic authentication"""
cli, mock_root, mock_connection = setup_basic_auth()

View File

@@ -1,7 +1,5 @@
import pytest
from requests.exceptions import ConnectionError
import sys
from unittest.mock import patch
from awxkit.cli import run, CLI
@@ -55,288 +53,3 @@ def test_list_resources(capfd, resource):
assert "usage:" in out
for snippet in ('--conf.host https://example.awx.org]', '-v, --verbose'):
assert snippet in out
class TestHelpHandling:
"""Test suite for improved help handling functionality"""
def test_get_non_option_args_basic(self):
"""Test _get_non_option_args extracts non-option arguments correctly"""
cli = CLI()
cli.argv = ['awx', 'users', 'list', '--verbose']
result = cli._get_non_option_args()
assert result == ['users', 'list']
def test_get_non_option_args_with_flags(self):
"""Test _get_non_option_args ignores option flags and their values"""
cli = CLI()
cli.argv = ['awx', '--conf.host', 'example.com', 'jobs', 'create', '--name', 'test']
result = cli._get_non_option_args()
# Should only include positional arguments, not option values
assert result == ['jobs', 'create']
def test_get_non_option_args_before_help(self):
"""Test _get_non_option_args with before_help=True stops at help flag"""
cli = CLI()
cli.argv = ['awx', 'users', '--help', 'extra', 'args']
result = cli._get_non_option_args(before_help=True)
assert result == ['users']
def test_get_non_option_args_before_help_short_flag(self):
"""Test _get_non_option_args with before_help=True stops at -h flag"""
cli = CLI()
cli.argv = ['awx', 'projects', '-h', 'should', 'not', 'appear']
result = cli._get_non_option_args(before_help=True)
assert result == ['projects']
def test_get_non_option_args_no_help_flag(self):
"""Test _get_non_option_args when help flag not present"""
cli = CLI()
cli.argv = ['awx', 'organizations', 'list']
result = cli._get_non_option_args(before_help=True)
assert result == ['organizations', 'list']
def test_is_main_help_request_true(self):
"""Test _is_main_help_request returns True for main CLI help"""
cli = CLI()
cli.argv = ['awx', '--help']
result = cli._is_main_help_request()
assert result is True
def test_is_main_help_request_short_flag(self):
"""Test _is_main_help_request returns True for main CLI help with -h"""
cli = CLI()
cli.argv = ['awx', '-h']
result = cli._is_main_help_request()
assert result is True
def test_is_main_help_request_false_subcommand(self):
"""Test _is_main_help_request returns False for subcommand help"""
cli = CLI()
cli.argv = ['awx', 'users', '--help']
result = cli._is_main_help_request()
assert result is False
def test_is_main_help_request_false_action(self):
"""Test _is_main_help_request returns False for action help"""
cli = CLI()
cli.argv = ['awx', 'jobs', 'create', '--help']
result = cli._is_main_help_request()
assert result is False
def test_is_main_help_request_false_no_help(self):
"""Test _is_main_help_request returns False when no help flag"""
cli = CLI()
cli.argv = ['awx', 'users', 'list']
result = cli._is_main_help_request()
assert result is False
def test_early_help_return_main_cli(self):
"""Test that main CLI help exits early without server connection"""
cli = CLI()
# Verify that _is_main_help_request works correctly
cli.argv = ['awx', '--help']
assert cli._is_main_help_request() is True
# Test that parse_args with main help flag should exit
with patch.object(sys, 'exit') as mock_exit:
cli.parse_args(['awx', '--help'])
mock_exit.assert_called_once_with(0)
def test_no_early_exit_for_subcommand_help(self):
"""Test that subcommand help does not exit early"""
with patch.object(sys, 'exit') as mock_exit:
cli = CLI()
# This should not exit early since it's subcommand help
cli.parse_args(['awx', 'users', '--help'])
mock_exit.assert_not_called()
def test_help_property_detection(self):
"""Test that help property correctly detects help flags"""
cli = CLI()
cli.argv = ['awx', '--help']
assert cli.help is True
cli.argv = ['awx', '-h']
assert cli.help is True
cli.argv = ['awx', 'users', '--help']
assert cli.help is True
cli.argv = ['awx', 'users', 'list']
assert cli.help is False
def test_short_help_flag_added(self):
"""Test that -h flag is properly added to argument parser"""
cli = CLI()
cli.parse_args(['awx'])
# Verify that both -h and --help are recognized
help_actions = [action for action in cli.parser._actions if '--help' in action.option_strings]
assert len(help_actions) == 1
assert '-h' in help_actions[0].option_strings
assert '--help' in help_actions[0].option_strings
def test_get_non_option_args_with_equals_format(self):
"""Test _get_non_option_args handles --opt=val format correctly"""
cli = CLI()
cli.argv = ['awx', 'users', 'create', '--email=john@example.com', '--name=john']
result = cli._get_non_option_args()
assert result == ['users', 'create']
def test_get_non_option_args_mixed_option_formats(self):
"""Test _get_non_option_args handles mixed --opt=val and --opt val formats"""
cli = CLI()
cli.argv = ['awx', 'jobs', 'launch', '--job-template=5', '--extra-vars', '{"key": "value"}', 'extra_arg']
result = cli._get_non_option_args()
assert result == ['jobs', 'launch', 'extra_arg']
def test_get_non_option_args_short_options(self):
"""Test _get_non_option_args handles short options correctly"""
cli = CLI()
cli.argv = ['awx', '-v', '-f', 'json', 'projects', 'list']
result = cli._get_non_option_args()
assert result == ['projects', 'list']
def test_get_non_option_args_consecutive_options(self):
"""Test _get_non_option_args with consecutive options"""
cli = CLI()
cli.argv = ['awx', '--conf.host', 'example.com', '--conf.username', 'admin', 'teams', 'create']
result = cli._get_non_option_args()
assert result == ['teams', 'create']
def test_get_non_option_args_option_at_end(self):
"""Test _get_non_option_args with option at the end"""
cli = CLI()
cli.argv = ['awx', 'users', 'list', '--format', 'table']
result = cli._get_non_option_args()
assert result == ['users', 'list']
def test_get_non_option_args_flag_only_options(self):
"""Test _get_non_option_args with flag-only options (no values)"""
cli = CLI()
# More realistic: flags at the end or grouped together
cli.argv = ['awx', 'organizations', 'list', '--verbose', '--insecure', '--monitor']
result = cli._get_non_option_args()
assert result == ['organizations', 'list']
def test_get_non_option_args_option_value_looks_like_option(self):
"""Test _get_non_option_args when option value starts with dash"""
cli = CLI()
cli.argv = ['awx', 'jobs', 'create', '--description', '-some-description-with-dashes', 'template']
result = cli._get_non_option_args()
# Values starting with '-' are treated as options, and 'template' becomes the value for that "option"
# Users should use --description="-some-value" format for values starting with dash
assert result == ['jobs', 'create']
def test_get_non_option_args_complex_scenario(self):
"""Test _get_non_option_args with complex mixed arguments"""
cli = CLI()
cli.argv = [
'awx',
'--conf.host=https://example.com',
'job_templates',
'create',
'--name',
'my-template',
'--job-type=run',
'--inventory',
'1',
'--project=2',
'--verbose',
]
result = cli._get_non_option_args()
assert result == ['job_templates', 'create']
def test_get_non_option_args_before_help_with_options(self):
"""Test _get_non_option_args before_help=True with options before help"""
cli = CLI()
cli.argv = ['awx', '--conf.host', 'example.com', 'users', 'create', '--name=test', '--help', 'ignored']
result = cli._get_non_option_args(before_help=True)
assert result == ['users', 'create']
def test_get_non_option_args_before_help_only_options(self):
"""Test _get_non_option_args before_help=True with only options before help"""
cli = CLI()
cli.argv = ['awx', '--verbose', '--conf.host=example.com', '--help', 'users', 'list']
result = cli._get_non_option_args(before_help=True)
assert result == []
def test_is_main_help_request_with_options_before_help(self):
"""Test _is_main_help_request with options but no subcommands before help"""
cli = CLI()
cli.argv = ['awx', '--conf.host=example.com', '--verbose', '--help']
result = cli._is_main_help_request()
assert result is True
def test_is_main_help_request_false_with_subcommand_and_options(self):
"""Test _is_main_help_request returns False when subcommand present with options"""
cli = CLI()
cli.argv = ['awx', '--conf.host', 'example.com', 'users', '--format=json', '--help']
result = cli._is_main_help_request()
assert result is False
def test_is_main_help_request_false_option_value_looks_like_subcommand(self):
"""Test _is_main_help_request doesn't mistake option values for subcommands"""
cli = CLI()
cli.argv = ['awx', '--conf.host', 'users', '--help'] # 'users' is option value, not subcommand
result = cli._is_main_help_request()
assert result is True # Should be True since 'users' is just an option value
def test_is_main_help_request_complex_option_scenario(self):
"""Test _is_main_help_request with complex option scenario"""
cli = CLI()
cli.argv = ['awx', '--conf.username=admin', '--conf.password', 'secret', 'job_templates', '--help']
result = cli._is_main_help_request()
assert result is False # 'job_templates' is a real subcommand, not an option value
def test_empty_args_handling(self):
"""Test _get_non_option_args handles minimal arguments"""
cli = CLI()
cli.argv = ['awx']
result = cli._get_non_option_args()
assert result == []
def test_only_awx_and_options(self):
"""Test _get_non_option_args with only awx and options"""
cli = CLI()
cli.argv = ['awx', '--verbose', '--conf.host=example.com']
result = cli._get_non_option_args()
assert result == []
def test_get_non_option_args_dash_value_with_equals(self):
"""Test _get_non_option_args handles dash values correctly with equals format"""
cli = CLI()
cli.argv = ['awx', 'jobs', 'create', '--description=-some-description-with-dashes', 'template']
result = cli._get_non_option_args()
# Using --opt=val format correctly handles values starting with dash
assert result == ['jobs', 'create', 'template']

View File

@@ -251,24 +251,3 @@ class TestSettingsOptions(unittest.TestCase):
out = StringIO()
self.parser.choices['modify'].print_help(out)
assert 'modify [-h] key value' in out.getvalue()
class TestHelpParameterChanges(unittest.TestCase):
"""Test that add_help parameter changes work correctly"""
def test_add_help_parameter_handling(self):
"""Test that add_help=True and add_help=False work as expected"""
# Test add_help=True (for action parsers like list, create)
parser = argparse.ArgumentParser()
subparsers = parser.add_subparsers(dest='action')
list_parser = subparsers.add_parser('list', help='', add_help=True)
out = StringIO()
list_parser.print_help(out)
help_text = out.getvalue()
assert 'show this help message and exit' in help_text
# Test add_help=False (for resource parsers like users, jobs)
resource_parser = subparsers.add_parser('users', help='', add_help=False)
help_actions = [action for action in resource_parser._actions if '--help' in action.option_strings]
assert len(help_actions) == 0 # Should be 0 because add_help=False

View File

@@ -1,195 +0,0 @@
# External Query Files for Indirect Node Counting
This document describes how to create query files for the Indirect Node Counting feature. Query files define how to extract managed node information from Ansible module execution results.
## Overview
When Ansible modules interact with external systems (VMware, cloud providers, network devices, etc.), they may manage nodes that aren't in the Ansible inventory. Query files tell the Controller how to extract information about these "indirect" managed nodes from module execution data.
## Query File Types
There are two types of query files:
1. **Embedded Query Files**: Shipped within a collection at `extensions/audit/event_query.yml`
2. **External Query Files**: Shipped in the `redhat.indirect_accounting` collection at `extensions/audit/external_queries/<namespace>.<name>.<version>.yml`
Embedded queries take precedence over external queries. External queries support version fallback within the same major version.
## File Format
Query files are YAML documents that map fully-qualified module names to jq expressions.
### Basic Structure
```yaml
---
<namespace>.<collection>.<module_name>:
query: >-
<jq_expression>
```
### Example
```yaml
---
community.vmware.vmware_guest:
query: >-
{name: .instance.hw_name, canonical_facts: {host_name: .instance.hw_name, uuid: .instance.hw_product_uuid}, facts: {guest_id: .instance.hw_guest_id}}
```
## jq Expression Requirements
The jq expression processes the module's result data (`event_data.res`) and must output a JSON object with the following fields:
### Required Fields
| Field | Type | Description |
|-------|------|-------------|
| `name` | string | Display name of the indirect managed node |
| `canonical_facts` | object | Facts used for node deduplication across jobs |
### Optional Fields
| Field | Type | Description |
|-------|------|-------------|
| `facts` | object | Additional information about the managed node |
### canonical_facts
The `canonical_facts` object should contain fields that uniquely identify the managed node. Common examples:
- `host_name`: The hostname of the managed node
- `uuid`: A unique identifier (VM UUID, device serial number, etc.)
- `ip_address`: IP address if it uniquely identifies the node
These facts are used to deduplicate nodes across multiple job runs. Choose facts that remain stable across the node's lifecycle.
### facts
The `facts` object contains additional metadata that doesn't affect deduplication:
- `device_type`: Type of device (e.g., "virtual_machine", "network_switch")
- `guest_id`: Guest OS identifier
- `platform`: Platform information
## jq Expression Input
The jq expression receives the module's result data as input. This is the `res` field from Ansible's job event data, which typically contains:
- The module's return values
- Any registered variables
- Status information
To understand what data is available, examine the module's documentation or run a test playbook and inspect the job events.
## Module Matching
### Exact Match
Queries are matched by fully-qualified module name:
```yaml
community.vmware.vmware_guest:
query: >-
...
```
This matches only `community.vmware.vmware_guest` module invocations.
### Wildcard Match
You can use wildcards to match all modules in a collection:
```yaml
community.vmware.*:
query: >-
...
```
Exact matches take precedence over wildcard matches.
## External Query File Naming
External query files must follow this naming convention:
```
<namespace>.<collection_name>.<version>.yml
```
Examples:
- `community.vmware.4.5.0.yml`
- `cisco.ios.8.0.0.yml`
- `amazon.aws.7.2.1.yml`
## Version Fallback
When no exact version match exists for an external query, the system falls back to the nearest compatible version:
1. Only versions with the **same major version** are considered
2. The **highest version less than or equal to** the installed version is selected
3. Major version boundaries are never crossed
### Examples
| Installed Version | Available Queries | Query Used | Reason |
|-------------------|-------------------|------------|--------|
| 4.5.0 | 4.0.0, 4.1.0, 5.0.0 | 4.1.0 | Highest v4.x <= 4.5.0 |
| 4.0.5 | 4.0.0, 4.1.0, 5.0.0 | 4.0.0 | 4.1.0 > 4.0.5, so 4.0.0 |
| 5.2.0 | 4.0.0, 4.1.0, 5.0.0 | 5.0.0 | Highest v5.x <= 5.2.0 |
| 3.8.0 | 4.0.0, 4.1.0, 5.0.0 | None | No v3.x queries available |
| 6.0.0 | 4.0.0, 4.1.0, 5.0.0 | None | No v6.x queries available |
## Complete Example
Here's a complete external query file for `community.vmware` version 4.5.0:
**File**: `extensions/audit/external_queries/community.vmware.4.5.0.yml`
```yaml
---
# Query for vmware_guest module - extracts VM information
community.vmware.vmware_guest:
query: >-
{name: .instance.hw_name, canonical_facts: {host_name: .instance.hw_name, uuid: .instance.hw_product_uuid}, facts: {guest_id: .instance.hw_guest_id, num_cpus: .instance.hw_processor_count}}
# Query for vmware_guest_info module
community.vmware.vmware_guest_info:
query: >-
{name: .instance.hw_name, canonical_facts: {host_name: .instance.hw_name, uuid: .instance.hw_product_uuid}, facts: {power_state: .instance.hw_power_status}}
```
## Testing Query Files
To test a query file:
1. Run a playbook that uses the target module
2. Examine the job events to see the module's result data
3. Test your jq expression against the result data using the `jq` command-line tool
4. Verify the output contains valid `name` and `canonical_facts` fields
Example testing with jq:
```bash
# Sample module result data (from job event)
echo '{"instance": {"hw_name": "test-vm", "hw_product_uuid": "abc-123"}}' | \
jq '{name: .instance.hw_name, canonical_facts: {host_name: .instance.hw_name, uuid: .instance.hw_product_uuid}}'
```
## Troubleshooting
### Query Not Being Applied
1. Verify the file is in the correct location
2. Check the file naming matches the collection namespace, name, and version exactly
3. Ensure the module name in the query matches the fully-qualified module name
### No Indirect Nodes Counted
1. Verify the jq expression produces valid output with `canonical_facts`
2. Check the Controller logs for jq parsing errors
3. Ensure the module's result data contains the expected fields
### Version Fallback Not Working
1. Verify the fallback version has the same major version as the installed collection
2. Check that the fallback version is less than or equal to the installed version

View File

@@ -15,6 +15,9 @@ markers =
filterwarnings =
error
# FIXME: Upgrade protobuf https://github.com/protocolbuffers/protobuf/issues/15077
once:Type google._upb._message.* uses PyType_Spec with a metaclass that has custom tp_new:DeprecationWarning
# FIXME: Upgrade python-dateutil https://github.com/dateutil/dateutil/issues/1340
once:datetime.datetime.utcfromtimestamp\(\) is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC:DeprecationWarning
@@ -28,12 +31,44 @@ filterwarnings =
# FIXME: Delete this entry once `zope` is updated.
once:Deprecated call to `pkg_resources.declare_namespace.'zope'.`.\nImplementing implicit namespace packages .as specified in PEP 420. is preferred to `pkg_resources.declare_namespace`. See https.//setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages:DeprecationWarning:
# FIXME: Delete this entry once `coreapi` is deleted from the dependencies
# FIXME: and is no longer imported at runtime.
once:CoreAPI compatibility is deprecated and will be removed in DRF 3.17:rest_framework.RemovedInDRF317Warning:rest_framework.schemas.coreapi
# FIXME: Delete this entry once naive dates aren't passed to DB lookup
# FIXME: methods. Not sure where, might be in awx's views or in DAB.
once:DateTimeField User.date_joined received a naive datetime .2020-01-01 00.00.00. while time zone support is active.:RuntimeWarning:django.db.models.fields
# FIXME: Delete this entry once the deprecation is acted upon.
# Note: RemovedInDjango51Warning may not exist in newer Django versions
ignore:'index_together' is deprecated. Use 'Meta.indexes' in 'main.\w+' instead.
# FIXME: Add ordering to Resource model in django-ansible-base and delete this entry.
once:Pagination may yield inconsistent results with an unordered object_list.*Resource:django.core.paginator.UnorderedObjectListWarning
# FIXME: Update `awx.main.migrations._dab_rbac` and delete this entry.
# Note: RemovedInDjango50Warning may not exist in newer Django versions
ignore:Using QuerySet.iterator.. after prefetch_related.. without specifying chunk_size is deprecated.
# FIXME: Delete this entry once the **broken** always-true assertions in the
# FIXME: following tests are fixed:
# * `awx/main/tests/unit/utils/test_common.py::TestHostnameRegexValidator::test_good_call`
# * `awx/main/tests/unit/utils/test_common.py::TestHostnameRegexValidator::test_bad_call_with_inverse`
once:assertion is always true, perhaps remove parentheses\?:pytest.PytestAssertRewriteWarning:
# FIXME: Figure this out, fix and then delete the entry. It's not entirely
# FIXME: clear what emits it and where.
once:Pagination may yield inconsistent results with an unordered object_list. .class 'awx.main.models.workflow.WorkflowJobTemplateNode'. QuerySet.:django.core.paginator.UnorderedObjectListWarning:django.core.paginator
# FIXME: Figure this out, fix and then delete the entry.
once::django.core.paginator.UnorderedObjectListWarning:rest_framework.pagination
# FIXME: Use `codecs.open()` via a context manager
# FIXME: in `awx/main/utils/ansible.py` to close hanging file descriptors
# FIXME: and then delete the entry.
once:unclosed file <_io.BufferedReader name='[^']+'>:ResourceWarning:awx.main.utils.ansible
# FIXME: Use `open()` via a context manager
# FIXME: in `awx/main/tests/unit/test_tasks.py` to close hanging file
# FIXME: descriptors and then delete the entry.
once:unclosed file <_io.TextIOWrapper name='[^']+' mode='r' encoding='UTF-8'>:ResourceWarning:awx.main.tests.unit.test_tasks
# https://docs.pytest.org/en/stable/usage.html#creating-junitxml-format-files
junit_duration_report = call

View File

@@ -76,3 +76,6 @@ django-flags>=5.0.13
dispatcherd[pg-notify] # tasking system, previously part of AWX code base
protobuf>=4.25.8 # CVE-2025-4565
idna>=3.10 # CVE-2024-3651
# Temporarily added to use ansible-runner from git branch, to be removed
# when ansible-runner moves from requirements_git.txt to here
pbr

View File

@@ -116,7 +116,7 @@ cython==3.1.3
# via -r /awx_devel/requirements/requirements.in
daphne==4.2.1
# via -r /awx_devel/requirements/requirements.in
dispatcherd[pg-notify]==2026.02.26
dispatcherd[pg-notify]==2026.01.27
# via -r /awx_devel/requirements/requirements.in
distro==1.9.0
# via -r /awx_devel/requirements/requirements.in
@@ -337,6 +337,8 @@ packaging==25.0
# opentelemetry-instrumentation
# setuptools-scm
# wheel
pbr==7.0.1
# via -r /awx_devel/requirements/requirements.in
pexpect==4.9.0
# via
# -r /awx_devel/requirements/requirements.in
@@ -554,5 +556,6 @@ setuptools==80.9.0
# -r /awx_devel/requirements/requirements.in
# autobahn
# incremental
# pbr
# setuptools-rust
# setuptools-scm

View File

@@ -21,7 +21,7 @@ sonar.projectName=awx
# Source directories to analyze
sonar.sources=.
sonar.inclusions=awx/**,awxkit/**
sonar.inclusions=awx/**
# Test directories
sonar.tests=awx/main/tests
@@ -54,7 +54,7 @@ sonar.sourceEncoding=UTF-8
# =============================================================================
# Test and coverage reports (paths relative to project root)
sonar.python.coverage.reportPaths=reports/coverage.xml,awxkit/coverage.xml
sonar.python.coverage.reportPaths=reports/coverage.xml
sonar.python.xunit.reportPath=/reports/junit.xml
# External tool reports (add these paths when tools are configured)

View File

@@ -100,7 +100,7 @@ services:
- "3000:3001" # used by the UI dev env
{% endif %}
redis_{{ container_postfix }}:
image: mirror.gcr.io/library/redis:7.4.8
image: mirror.gcr.io/library/redis:latest
container_name: tools_redis_{{ container_postfix }}
volumes:
- "../../redis/redis.conf:/usr/local/etc/redis/redis.conf:Z"
@@ -112,8 +112,6 @@ services:
{% endfor %}
{% if control_plane_node_count|int > 1 %}
haproxy:
# NOTE: upgrading past 2.3 requires updating haproxy.cfg (deprecated httpchk syntax)
# and may require adding 'no strict-limits' to the global section.
image: mirror.gcr.io/library/haproxy:2.3
user: "{{ ansible_user_uid }}"
volumes:
@@ -132,8 +130,7 @@ services:
{% endif %}
{% if enable_splunk|bool %}
splunk:
# NOTE: upgrading to 10.x requires adding SPLUNK_GENERAL_TERMS: --accept-sgt-current-at-splunk-com
image: mirror.gcr.io/splunk/splunk:9.4.2
image: mirror.gcr.io/splunk/splunk:latest
container_name: tools_splunk_1
hostname: splunk
networks:
@@ -148,7 +145,7 @@ services:
{% endif %}
{% if enable_prometheus|bool %}
prometheus:
image: mirror.gcr.io/prom/prometheus:v3.10.0
image: mirror.gcr.io/prom/prometheus:latest
container_name: tools_prometheus_1
hostname: prometheus
networks:
@@ -161,7 +158,7 @@ services:
{% endif %}
{% if enable_grafana|bool %}
grafana:
image: mirror.gcr.io/grafana/grafana-enterprise:12.3.4
image: mirror.gcr.io/grafana/grafana-enterprise:latest
container_name: tools_grafana_1
hostname: grafana
networks:
@@ -202,7 +199,7 @@ services:
- "${AWX_PG_PORT:-5441}:5432"
{% if enable_pgbouncer|bool %}
pgbouncer:
image: mirror.gcr.io/bitnami/pgbouncer:1.24.0
image: mirror.gcr.io/bitnami/pgbouncer:latest
container_name: tools_pgbouncer_1
hostname: pgbouncer
networks:

View File

@@ -43,13 +43,13 @@ From root of AWX project `~/projects/src/github.com/TheRealHaoLiu/awx`
I can either do
```bash
ln -s ~/projects/src/github.com/TheRealHaoLiu/django-ansible-base tools/docker-compose/editable_dependencies/
ln -s ~/projects/src/github.com/TheRealHaoLiu/ansible-runner tools/docker-compose/editable_dependencies/
```
or
```bash
ln -s ../django-ansible-base tools/docker-compose/editable_dependencies/
ln -s ../ansible-runner tools/docker-compose/editable_dependencies/
```
## How to remove indivisual editable dependencies