Compare commits

..

16 Commits

Author SHA1 Message Date
thedoubl3j
0377b3830b Update operator timeout
* updated the operator timeout to near healthy run time
2026-01-23 10:39:55 -05:00
Jake Jackson
331ae92475 Merge branch 'devel' into move_to_dispatcherd 2026-01-23 10:06:30 -05:00
Jake Jackson
e355df6cc6 Merge branch 'devel' into move_to_dispatcherd 2026-01-22 11:03:45 -05:00
thedoubl3j
806ef7c345 Fix attribute error in server logs
* on a secret hunt to find the hidden attribute error in the server logs
2026-01-20 16:08:46 -05:00
thedoubl3j
8acdd0cbf4 Fix imports and linter findings
* add back more missing things
2026-01-20 15:41:33 -05:00
thedoubl3j
381c7fdc5d Adjust heartbeat arg and more formatting
* fixed the call to cluster_node_heartbeat missing binder
* formatting/linter fixes
2026-01-20 15:21:23 -05:00
thedoubl3j
d75fcc13f6 Fix dispatcher run call and remove dispatch settin
* added back some code that was lost in the merge conflict
* remove dispatcher mock publish setting
2026-01-20 14:38:54 -05:00
thedoubl3j
bb8ecc5919 Add back hazmat for config and remove baseworker
* added back hazmat per @alancoding feedback around config
* removed baseworker completely and refactored it into the callback
  worker
2026-01-19 20:33:23 -05:00
thedoubl3j
1019ac0439 Update function comments 2026-01-19 20:30:41 -05:00
thedoubl3j
cddee29f23 More chainsaw work
* fixed imports and addressed clusternode heartbeat test
* took a chainsaw to task.py as well
2026-01-19 20:30:41 -05:00
thedoubl3j
3b896a00a9 Clean up imports and fix some tests
* removed unused imports
* adjusted test import to pull correct method
2026-01-19 20:30:41 -05:00
thedoubl3j
e386326498 Remove control and hazmat (squash this not done)
* moved status out and deleted control as no longer needed
* removed hazmat
2026-01-19 20:30:41 -05:00
thedoubl3j
5209bfcf82 add back auto_max_workers
* added back get_auto_max_workers into common utils
* formatting edits
2026-01-19 20:30:07 -05:00
thedoubl3j
ebd51cd074 Keep callback receiver working
* remove any code that is not used by the call back receiver
2026-01-19 20:26:04 -05:00
thedoubl3j
f9f4bf2d1a Add decorator
* moved to dispatcher decorator
* updated as many as I could find
2026-01-19 20:26:04 -05:00
thedoubl3j
e55578b64e WIP First pass
* started removing feature flags and adjusting logic
* WIP
2026-01-19 20:26:04 -05:00
222 changed files with 1401 additions and 12737 deletions

View File

@@ -24,7 +24,7 @@ in as the first entry for your PR title.
##### STEPS TO REPRODUCE AND EXTRA INFO
##### ADDITIONAL INFORMATION
<!---
Include additional information to help people understand the change here.
For bugs that don't have a linked bug report, a step-by-step reproduction

View File

@@ -1,55 +0,0 @@
---
name: Repo Owns Branch
# Reusable workflow that determines whether the current repository
# owns the current branch for push operations.
#
# Ownership rules:
# - ansible/awx owns: devel, feature_*
# - ansible/tower owns: stable-*, release_*
# - workflow_dispatch is always allowed
#
# All other repo/branch combinations are skipped.
on:
workflow_call:
outputs:
should_run:
description: Whether this repo owns the current branch
value: ${{ jobs.check.outputs.should_run }}
jobs:
check:
runs-on: ubuntu-latest
outputs:
should_run: ${{ steps.check.outputs.should_run }}
steps:
- name: Check branch ownership
id: check
run: |
REPO="${{ github.repository }}"
BRANCH="${{ github.ref_name }}"
EVENT="${{ github.event_name }}"
if [[ "$EVENT" == "workflow_dispatch" ]]; then
echo "should_run=true" >> $GITHUB_OUTPUT
echo "Manual trigger — allowed"
exit 0
fi
# ansible/awx owns devel and feature_* branches
if [[ "$REPO" == "ansible/awx" ]] && [[ "$BRANCH" == "devel" || "$BRANCH" == feature_* ]]; then
echo "should_run=true" >> $GITHUB_OUTPUT
echo "Repository '$REPO' owns branch '$BRANCH'"
exit 0
fi
# ansible/tower owns stable-* and release_* branches
if [[ "$REPO" == "ansible/tower" ]] && [[ "$BRANCH" == stable-* || "$BRANCH" == release_* ]]; then
echo "should_run=true" >> $GITHUB_OUTPUT
echo "Repository '$REPO' owns branch '$BRANCH'"
exit 0
fi
echo "should_run=false" >> $GITHUB_OUTPUT
echo "Repository '$REPO' does not own branch '$BRANCH' — skipping"

View File

@@ -45,45 +45,15 @@ jobs:
make docker-runner 2>&1 | tee schema-diff.txt
exit ${PIPESTATUS[0]}
- name: Validate OpenAPI schema
id: schema-validation
continue-on-error: true
run: |
AWX_DOCKER_ARGS='-e GITHUB_ACTIONS' \
AWX_DOCKER_CMD='make validate-openapi-schema' \
make docker-runner 2>&1 | tee schema-validation.txt
exit ${PIPESTATUS[0]}
- name: Add schema validation and diff to job summary
- name: Add schema diff to job summary
if: always()
# show text and if for some reason, it can't be generated, state that it can't be.
run: |
echo "## API Schema Check Results" >> $GITHUB_STEP_SUMMARY
echo "## API Schema Change Detection Results" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Show validation status
echo "### OpenAPI Validation" >> $GITHUB_STEP_SUMMARY
if [ -f schema-validation.txt ] && grep -q "✓ Schema is valid" schema-validation.txt; then
echo "✅ **Status:** PASSED - Schema is valid OpenAPI 3.0.3" >> $GITHUB_STEP_SUMMARY
else
echo "❌ **Status:** FAILED - Schema validation failed" >> $GITHUB_STEP_SUMMARY
if [ -f schema-validation.txt ]; then
echo "" >> $GITHUB_STEP_SUMMARY
echo "<details><summary>Validation errors</summary>" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
cat schema-validation.txt >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
echo "</details>" >> $GITHUB_STEP_SUMMARY
fi
fi
echo "" >> $GITHUB_STEP_SUMMARY
# Show schema changes
echo "### Schema Changes" >> $GITHUB_STEP_SUMMARY
if [ -f schema-diff.txt ]; then
if grep -q "^+" schema-diff.txt || grep -q "^-" schema-diff.txt; then
echo "**Changes detected** between this PR and the base branch" >> $GITHUB_STEP_SUMMARY
echo "### Schema changes detected" >> $GITHUB_STEP_SUMMARY
echo "" >> $GITHUB_STEP_SUMMARY
# Truncate to first 1000 lines to stay under GitHub's 1MB summary limit
TOTAL_LINES=$(wc -l < schema-diff.txt)
@@ -95,8 +65,8 @@ jobs:
head -n 1000 schema-diff.txt >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
else
echo "No schema changes detected" >> $GITHUB_STEP_SUMMARY
echo "### No schema changes detected" >> $GITHUB_STEP_SUMMARY
fi
else
echo "Unable to generate schema diff" >> $GITHUB_STEP_SUMMARY
echo "### Unable to generate schema diff" >> $GITHUB_STEP_SUMMARY
fi

View File

@@ -4,46 +4,14 @@ env:
LC_ALL: "C.UTF-8" # prevent ERROR: Ansible could not initialize the preferred locale: unsupported locale setting
CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
DEV_DOCKER_OWNER: ${{ github.repository_owner }}
COMPOSE_TAG: ${{ github.base_ref || github.ref_name || 'devel' }}
COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
UPSTREAM_REPOSITORY_ID: 91594105
on:
pull_request:
push:
branches:
- devel # needed to publish code coverage post-merge
schedule:
- cron: '0 12,18 * * 1-5'
workflow_dispatch: {}
jobs:
trigger-release-branches:
name: "Dispatch CI to release branches"
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
permissions:
actions: write
steps:
- name: Trigger CI on release_4.6
id: dispatch_release_46
continue-on-error: true
run: gh workflow run ci.yml --ref release_4.6
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
- name: Trigger CI on stable-2.6
id: dispatch_stable_26
continue-on-error: true
run: gh workflow run ci.yml --ref stable-2.6
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
GH_REPO: ${{ github.repository }}
- name: Check dispatch results
if: steps.dispatch_release_46.outcome == 'failure' || steps.dispatch_stable_26.outcome == 'failure'
run: |
echo "One or more dispatches failed:"
echo " release_4.6: ${{ steps.dispatch_release_46.outcome }}"
echo " stable-2.6: ${{ steps.dispatch_stable_26.outcome }}"
exit 1
common-tests:
name: ${{ matrix.tests.name }}
runs-on: ubuntu-latest
@@ -94,11 +62,7 @@ jobs:
run: |
if [ -f "reports/coverage.xml" ]; then
sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' reports/coverage.xml
echo "Injected PR number ${{ github.event.pull_request.number }} into reports/coverage.xml"
fi
if [ -f "awxkit/coverage.xml" ]; then
sed -i '2i<!-- PR ${{ github.event.pull_request.number }} -->' awxkit/coverage.xml
echo "Injected PR number ${{ github.event.pull_request.number }} into awxkit/coverage.xml"
echo "Injected PR number ${{ github.event.pull_request.number }} into coverage.xml"
fi
- name: Upload test coverage to Codecov
@@ -145,32 +109,28 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.tests.name }}-artifacts
path: |
reports/coverage.xml
awxkit/coverage.xml
path: reports/coverage.xml
retention-days: 5
- name: >-
Upload ${{
matrix.tests.coverage-upload-name || 'awx'
}} jUnit test reports to the unified dashboard
- name: Upload awx jUnit test reports
if: >-
!cancelled()
&& steps.make-run.outputs.test-result-files != ''
&& github.event_name == 'push'
&& env.UPSTREAM_REPOSITORY_ID == github.repository_id
&& github.ref_name == github.event.repository.default_branch
uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
with:
aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
http-auth-password: >-
${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}
http-auth-username: >-
${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}
project-component-name: >-
${{ matrix.tests.coverage-upload-name || 'awx' }}
test-result-files: >-
${{ steps.make-run.outputs.test-result-files }}
run: |
for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
do
curl \
-v \
--user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
--form "xunit_xml=@${junit_file}" \
--form "component_name=${{ matrix.tests.coverage-upload-name || 'awx' }}" \
--form "git_commit_sha=${{ github.sha }}" \
--form "git_repository_url=https://github.com/${{ github.repository }}" \
"${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
done
dev-env:
runs-on: ubuntu-latest
@@ -334,16 +294,18 @@ jobs:
&& github.event_name == 'push'
&& env.UPSTREAM_REPOSITORY_ID == github.repository_id
&& github.ref_name == github.event.repository.default_branch
uses: ansible/gh-action-record-test-results@3784db66a1b7fb3809999a7251c8a7203a7ffbe8
with:
aggregation-server-url: ${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}
http-auth-password: >-
${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}
http-auth-username: >-
${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}
project-component-name: awx
test-result-files: >-
${{ steps.make-run.outputs.test-result-files }}
run: |
for junit_file in $(echo '${{ steps.make-run.outputs.test-result-files }}' | sed 's/,/ /')
do
curl \
-v \
--user "${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_USER }}:${{ secrets.PDE_ORG_RESULTS_UPLOAD_PASSWORD }}" \
--form "xunit_xml=@${junit_file}" \
--form "component_name=awx" \
--form "git_commit_sha=${{ github.sha }}" \
--form "git_repository_url=https://github.com/${{ github.repository }}" \
"${{ vars.PDE_ORG_RESULTS_AGGREGATOR_UPLOAD_URL }}/api/results/upload/"
done
collection-integration:
name: awx_collection integration

View File

@@ -12,12 +12,7 @@ on:
- feature_*
- stable-*
jobs:
check-ownership:
uses: ./.github/workflows/_repo-owns-branch.yml
push-development-images:
needs: check-ownership
if: needs.check-ownership.outputs.should_run == 'true'
runs-on: ubuntu-latest
timeout-minutes: 120
permissions:
@@ -35,6 +30,12 @@ jobs:
make-target: awx-kube-buildx
steps:
- name: Skipping build of awx image for non-awx repository
run: |
echo "Skipping build of awx image for non-awx repository"
exit 0
if: matrix.build-targets.image-name == 'awx' && !endsWith(github.repository, '/awx')
- uses: actions/checkout@v4
with:
show-progress: false

View File

@@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
timeout-minutes: 20
permissions:
packages: read
packages: write
contents: read
steps:
- name: Check for each of the lines

View File

@@ -1,183 +0,0 @@
# Sync OpenAPI Spec on Merge
#
# This workflow runs when code is merged to the devel branch.
# It runs the dev environment to generate the OpenAPI spec, then syncs it to
# the central spec repository.
#
# FLOW: PR merged → push to branch → dev environment runs → spec synced to central repo
#
# NOTE: This is an inlined version for testing with private forks.
# Production version will use a reusable workflow from the org repos.
name: Sync OpenAPI Spec on Merge
env:
LC_ALL: "C.UTF-8"
DEV_DOCKER_OWNER: ${{ github.repository_owner }}
on:
push:
branches:
- devel
- 'stable-2.[6-9]'
- 'stable-2.[1-9][0-9]'
workflow_dispatch: # Allow manual triggering for testing
jobs:
check-ownership:
uses: ./.github/workflows/_repo-owns-branch.yml
sync-openapi-spec:
needs: check-ownership
if: needs.check-ownership.outputs.should_run == 'true'
name: Sync OpenAPI spec to central repo
runs-on: ubuntu-latest
permissions:
packages: write
contents: read
steps:
- name: Checkout Controller repository
uses: actions/checkout@v4
with:
show-progress: false
- name: Build awx_devel image to use for schema gen
uses: ./.github/actions/awx_devel_image
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
private-github-key: ${{ secrets.PRIVATE_GITHUB_KEY }}
- name: Generate API Schema
run: |
DEV_DOCKER_TAG_BASE=ghcr.io/${OWNER_LC} \
COMPOSE_TAG=${{ github.base_ref || github.ref_name }} \
docker run -u $(id -u) --rm -v ${{ github.workspace }}:/awx_devel/:Z \
--workdir=/awx_devel `make print-DEVEL_IMAGE_NAME` /start_tests.sh genschema
- name: Verify spec file exists
run: |
SPEC_FILE="./schema.json"
if [ ! -f "$SPEC_FILE" ]; then
echo "❌ Spec file not found at $SPEC_FILE"
echo "Contents of workspace:"
ls -la .
exit 1
fi
echo "✅ Found spec file at $SPEC_FILE"
- name: Checkout spec repo
id: checkout_spec_repo
continue-on-error: true
uses: actions/checkout@v4
with:
repository: ansible-automation-platform/aap-openapi-specs
ref: ${{ github.ref_name }}
path: spec-repo
token: ${{ secrets.OPENAPI_SPEC_SYNC_TOKEN }}
- name: Fail if branch doesn't exist
if: steps.checkout_spec_repo.outcome == 'failure'
run: |
echo "##[error]❌ Branch '${{ github.ref_name }}' does not exist in the central spec repository."
echo "##[error]Expected branch: ${{ github.ref_name }}"
echo "##[error]This branch must be created in the spec repo before specs can be synced."
exit 1
- name: Compare specs
id: compare
run: |
COMPONENT_SPEC="./schema.json"
SPEC_REPO_FILE="spec-repo/controller.json"
# Check if spec file exists in spec repo
if [ ! -f "$SPEC_REPO_FILE" ]; then
echo "Spec file doesn't exist in spec repo - will create new file"
echo "has_diff=true" >> $GITHUB_OUTPUT
echo "is_new_file=true" >> $GITHUB_OUTPUT
else
# Compare files
if diff -q "$COMPONENT_SPEC" "$SPEC_REPO_FILE" > /dev/null; then
echo "✅ No differences found - specs are identical"
echo "has_diff=false" >> $GITHUB_OUTPUT
else
echo "📝 Differences found - spec has changed"
echo "has_diff=true" >> $GITHUB_OUTPUT
echo "is_new_file=false" >> $GITHUB_OUTPUT
fi
fi
- name: Update spec file
if: steps.compare.outputs.has_diff == 'true'
run: |
cp "./schema.json" "spec-repo/controller.json"
echo "✅ Updated spec-repo/controller.json"
- name: Create PR in spec repo
if: steps.compare.outputs.has_diff == 'true'
working-directory: spec-repo
env:
GH_TOKEN: ${{ secrets.OPENAPI_SPEC_SYNC_TOKEN }}
COMMIT_MESSAGE: ${{ github.event.head_commit.message }}
run: |
# Configure git
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
# Create branch for PR
SHORT_SHA="${{ github.sha }}"
SHORT_SHA="${SHORT_SHA:0:7}"
BRANCH_NAME="update-Controller-${{ github.ref_name }}-${SHORT_SHA}"
git checkout -b "$BRANCH_NAME"
# Add and commit changes
git add "controller.json"
if [ "${{ steps.compare.outputs.is_new_file }}" == "true" ]; then
COMMIT_MSG="Add Controller OpenAPI spec for ${{ github.ref_name }}"
else
COMMIT_MSG="Update Controller OpenAPI spec for ${{ github.ref_name }}"
fi
git commit -m "$COMMIT_MSG
Synced from ${{ github.repository }}@${{ github.sha }}
Source branch: ${{ github.ref_name }}
Co-Authored-By: github-actions[bot] <github-actions[bot]@users.noreply.github.com>"
# Push branch
git push origin "$BRANCH_NAME"
# Create PR
PR_TITLE="[${{ github.ref_name }}] Update Controller spec from merged commit"
PR_BODY="## Summary
Automated OpenAPI spec sync from component repository merge.
**Source:** ${{ github.repository }}@${{ github.sha }}
**Branch:** \`${{ github.ref_name }}\`
**Component:** \`Controller\`
**Spec File:** \`controller.json\`
## Changes
$(if [ "${{ steps.compare.outputs.is_new_file }}" == "true" ]; then echo "- 🆕 New spec file created"; else echo "- 📝 Spec file updated with latest changes"; fi)
## Source Commit
\`\`\`
${COMMIT_MESSAGE}
\`\`\`
---
🤖 This PR was automatically generated by the OpenAPI spec sync workflow."
gh pr create \
--title "$PR_TITLE" \
--body "$PR_BODY" \
--base "${{ github.ref_name }}" \
--head "$BRANCH_NAME"
echo "✅ Created PR in spec repo"
- name: Report results
if: always()
run: |
if [ "${{ steps.compare.outputs.has_diff }}" == "true" ]; then
echo "📝 Spec sync completed - PR created in spec repo"
else
echo "✅ Spec sync completed - no changes needed"
fi

View File

@@ -13,12 +13,7 @@ on:
- feature_**
- stable-**
jobs:
check-ownership:
uses: ./.github/workflows/_repo-owns-branch.yml
push:
needs: check-ownership
if: needs.check-ownership.outputs.should_run == 'true'
runs-on: ubuntu-latest
timeout-minutes: 60
permissions:

View File

@@ -1,65 +0,0 @@
---
apiVersion: tekton.dev/v1
kind: PipelineRun
metadata:
name: awx-atf-tests-pull-request
annotations:
build.appstudio.openshift.io/repo: https://github.com/{{repo_owner}}/{{repo_name}}?rev={{revision}}
build.appstudio.redhat.com/commit_sha: '{{revision}}'
build.appstudio.redhat.com/pull_request_number: '{{pull_request_number}}'
build.appstudio.redhat.com/target_branch: '{{target_branch}}'
pipelinesascode.tekton.dev/cancel-in-progress: 'true'
pipelinesascode.tekton.dev/max-keep-runs: "3"
pipelinesascode.tekton.dev/on-comment: "^/run-atf-tests$"
pipelinesascode.tekton.dev/target-namespace: ansible-ci-tenant
labels:
appstudio.openshift.io/application: '{{repo_owner}}'
appstudio.openshift.io/component: '{{repo_owner}}-{{repo_name}}'
pipelines.appstudio.openshift.io/type: build
spec:
timeouts:
pipeline: "8h"
tasks: "7h"
finally: "1h"
pipelineRef:
resolver: bundles
params:
- name: name
value: aap-api-tests
- name: bundle
value: quay.io/aap-ci/tekton-catalog/pipeline/test/aap-api-tests:0.1@sha256:54d9e941748bae94b2154b3b253a985e628751dfa4508a138d9b05f74a3c1ddf
- name: kind
value: pipeline
- name: secret
value: quay-aap-ci-viewer
taskRunTemplate:
serviceAccountName: konflux-integration-runner
params:
- name: git-url
value: "{{source_url}}"
- name: pipeline-github-org
value: "{{repo_owner}}"
- name: pipeline-github-repo
value: "{{repo_name}}"
- name: pipeline-github-target-branch
value: '{{target_branch}}'
- name: pipeline-github-pr-revision
value: "{{revision}}"
- name: pipeline-github-pr-number
value: "{{pull_request_number}}"
- name: aap-dev-component-source-name
value: "controller"
- name: pytest-number-of-parallel-processes
value: "6"
workspaces:
- name: workspace
volumeClaimTemplate:
spec:
accessModes:
- ReadWriteOnce
resources:
requests:
storage: 1Gi

View File

@@ -103,12 +103,6 @@ When necessary, remove any AWX containers and images by running the following:
### Pre commit hooks
Install the pre-commit hook before contributing:
```
make pre-commit
```
When you attempt to perform a `git commit` there will be a pre-commit hook that gets run before the commit is allowed to your local repository. For example, python's [black](https://pypi.org/project/black/) will be run to test the formatting of any python files.
While you can use environment variables to skip the pre-commit hooks GitHub will run similar tests and prevent merging of PRs if the tests do not pass.

View File

@@ -1,6 +1,6 @@
-include awx/ui/Makefile
PYTHON := $(notdir $(shell for i in python3.12 python3.11 python3; do command -v $$i; done|sed 1q))
PYTHON := $(notdir $(shell for i in python3.12 python3; do command -v $$i; done|sed 1q))
SHELL := bash
DOCKER_COMPOSE ?= docker compose
OFFICIAL ?= no
@@ -10,7 +10,6 @@ KIND_BIN ?= $(shell which kind)
CHROMIUM_BIN=/tmp/chrome-linux/chrome
GIT_REPO_NAME ?= $(shell basename `git rev-parse --show-toplevel`)
GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
GIT_IS_WORKTREE := $(shell test -f .git && echo yes)
MANAGEMENT_COMMAND ?= awx-manage
VERSION ?= $(shell $(PYTHON) tools/scripts/scm_version.py 2> /dev/null)
@@ -80,7 +79,7 @@ RECEPTOR_IMAGE ?= quay.io/ansible/receptor:devel
SRC_ONLY_PKGS ?= cffi,pycparser,psycopg,twilio
# These should be upgraded in the AWX and Ansible venv before attempting
# to install the actual requirements
VENV_BOOTSTRAP ?= pip==25.3 setuptools==80.9.0 setuptools_scm[toml]==9.2.2 wheel==0.46.3 cython==3.1.3
VENV_BOOTSTRAP ?= pip==25.3 setuptools==80.9.0 setuptools_scm[toml]==9.2.2 wheel==0.45.1 cython==3.1.3
NAME ?= awx
@@ -107,15 +106,6 @@ else
DOCKER_KUBE_CACHE_FLAG=$(DOCKER_CACHE)
endif
# AWX TUI variables
AWX_HOST ?= https://localhost:8043
AWX_USER ?= admin
AWX_PASSWORD ?= $$(awk -F"'" '/^admin_password:/{print $$2}' tools/docker-compose/_sources/secrets/admin_password.yml 2>/dev/null || echo "admin")
AWX_VERIFY_SSL ?= false
# For git worktree to find the referenced git dir
GIT_COMMON_DIR := $(shell git rev-parse --git-common-dir 2>/dev/null || echo .git)
.PHONY: awx-link clean clean-tmp clean-venv requirements requirements_dev \
update_requirements upgrade_requirements update_requirements_dev \
docker_update_requirements docker_upgrade_requirements docker_update_requirements_dev \
@@ -123,7 +113,7 @@ GIT_COMMON_DIR := $(shell git rev-parse --git-common-dir 2>/dev/null || echo .gi
receiver test test_unit test_coverage coverage_html \
sdist \
VERSION PYTHON_VERSION docker-compose-sources \
pre-commit
.git/hooks/pre-commit
clean-tmp:
rm -rf tmp/
@@ -299,7 +289,7 @@ dispatcher:
@if [ "$(VENV_BASE)" ]; then \
. $(VENV_BASE)/awx/bin/activate; \
fi; \
$(PYTHON) manage.py dispatcherd
$(PYTHON) manage.py run_dispatcher
## Run to start the zeromq callback receiver
receiver:
@@ -352,10 +342,11 @@ black: reports
@command -v black >/dev/null 2>&1 || { echo "could not find black on your PATH, you may need to \`pip install black\`, or set AWX_IGNORE_BLACK=1" && exit 1; }
@(set -o pipefail && $@ $(BLACK_ARGS) awx awxkit awx_collection | tee reports/$@.report)
$(GIT_COMMON_DIR)/hooks/pre-commit:
ln -sf ../../pre-commit.sh $(GIT_COMMON_DIR)/hooks/pre-commit
pre-commit: $(GIT_COMMON_DIR)/hooks/pre-commit
.git/hooks/pre-commit:
@echo "if [ -x pre-commit.sh ]; then" > .git/hooks/pre-commit
@echo " ./pre-commit.sh;" >> .git/hooks/pre-commit
@echo "fi" >> .git/hooks/pre-commit
@chmod +x .git/hooks/pre-commit
genschema: awx-link reports
@if [ "$(VENV_BASE)" ]; then \
@@ -530,7 +521,7 @@ ifneq ($(ADMIN_PASSWORD),)
EXTRA_SOURCES_ANSIBLE_OPTS := -e admin_password=$(ADMIN_PASSWORD) $(EXTRA_SOURCES_ANSIBLE_OPTS)
endif
docker-compose-sources:
docker-compose-sources: .git/hooks/pre-commit
@if [ $(MINIKUBE_CONTAINER_GROUP) = true ]; then\
$(ANSIBLE_PLAYBOOK) -i tools/docker-compose/inventory -e minikube_setup=$(MINIKUBE_SETUP) tools/docker-compose-minikube/deploy.yml; \
fi;
@@ -562,7 +553,7 @@ docker-compose: awx/projects docker-compose-sources
$(MAKE) docker-compose-up
docker-compose-up:
$(if $(GIT_IS_WORKTREE),SETUPTOOLS_SCM_PRETEND_VERSION="$(VERSION)") $(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) up $(COMPOSE_UP_OPTS) --remove-orphans
docker-compose-down:
$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml $(COMPOSE_OPTS) down --remove-orphans
@@ -580,20 +571,6 @@ docker-compose-runtest: awx/projects docker-compose-sources
docker-compose-build-schema: awx/projects docker-compose-sources
$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml run --rm --service-ports --no-deps awx_1 make genschema
awx-tui:
@if ! command -v awx-tui > /dev/null 2>&1; then \
$(PYTHON) -m pip install awx-tui; \
fi
@if [ -f "$(HOME)/.config/awx-tui/config.yaml" ]; then \
$(PYTHON) -m awx_tui.main; \
else \
AWX_HOST=$(AWX_HOST) \
AWX_USER=$(AWX_USER) \
AWX_PASSWORD=$(AWX_PASSWORD) \
AWX_VERIFY_SSL=$(AWX_VERIFY_SSL) \
$(PYTHON) -m awx_tui.main --host $(AWX_HOST); \
fi
SCHEMA_DIFF_BASE_FOLDER ?= awx
SCHEMA_DIFF_BASE_BRANCH ?= devel
detect-schema-change: genschema
@@ -602,10 +579,6 @@ detect-schema-change: genschema
# diff exits with 1 when files differ - capture but don't fail
-diff -u -b reference-schema.json schema.json
validate-openapi-schema: genschema
@echo "Validating OpenAPI schema from schema.json..."
@python3 -c "from openapi_spec_validator import validate; import json; spec = json.load(open('schema.json')); validate(spec); print('✓ Schema is valid')"
docker-compose-clean: awx/projects
$(DOCKER_COMPOSE) -f tools/docker-compose/_sources/docker-compose.yml rm -sf

View File

@@ -89,7 +89,7 @@ class DeprecatedCredentialField(serializers.IntegerField):
def to_internal_value(self, pk):
try:
pk = int(pk)
except (ValueError, TypeError):
except ValueError:
self.fail('invalid')
try:
Credential.objects.get(pk=pk)

View File

@@ -131,14 +131,8 @@ class LoggedLoginView(auth_views.LoginView):
class LoggedLogoutView(auth_views.LogoutView):
# Override http_method_names to allow GET requests (Django 5.2+ defaults to POST only)
http_method_names = ["get", "post", "options"]
success_url_allowed_hosts = set(settings.LOGOUT_ALLOWED_HOSTS.split(",")) if settings.LOGOUT_ALLOWED_HOSTS else set()
def get(self, request, *args, **kwargs):
"""Handle GET requests for logout (for backward compatibility)."""
return self.post(request, *args, **kwargs)
def dispatch(self, request, *args, **kwargs):
if is_proxied_request():
# 1) We intentionally don't obey ?next= here, just always redirect to platform login
@@ -272,10 +266,7 @@ class APIView(views.APIView):
response = self.handle_exception(self.__init_request_error__)
if response.status_code == 401:
if response.data and 'detail' in response.data:
if getattr(settings, 'RESOURCE_SERVER__URL', None):
response.data['detail'] += _(' Direct access is not allowed, authenticate via the platform gateway.')
else:
response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
response.data['detail'] += _(' To establish a login session, visit') + ' /api/login/.'
logger.info(status_msg)
else:
logger.warning(status_msg)

View File

@@ -111,7 +111,7 @@ class UnifiedJobEventPagination(Pagination):
def __init__(self, *args, **kwargs):
self.use_limit_paginator = False
self.limit_pagination = LimitPagination()
super().__init__(*args, **kwargs)
return super().__init__(*args, **kwargs)
def paginate_queryset(self, queryset, request, view=None):
if 'limit' in request.query_params:

View File

@@ -9,50 +9,6 @@ from drf_spectacular.views import (
)
def filter_credential_type_schema(
result,
generator, # NOSONAR
request, # NOSONAR
public, # NOSONAR
):
"""
Postprocessing hook to filter CredentialType kind enum values.
For CredentialTypeRequest and PatchedCredentialTypeRequest schemas (POST/PUT/PATCH),
filter the 'kind' enum to only show 'cloud' and 'net' values.
This ensures the OpenAPI schema accurately reflects that only 'cloud' and 'net'
credential types can be created or modified via the API, matching the validation
in CredentialTypeSerializer.validate().
Args:
result: The OpenAPI schema dict to be modified
generator, request, public: Required by drf-spectacular interface (unused)
Returns:
The modified OpenAPI schema dict
"""
schemas = result.get('components', {}).get('schemas', {})
# Filter CredentialTypeRequest (POST/PUT) - field is required
if 'CredentialTypeRequest' in schemas:
kind_prop = schemas['CredentialTypeRequest'].get('properties', {}).get('kind', {})
if 'enum' in kind_prop:
# Filter to only cloud and net (no None - field is required)
kind_prop['enum'] = ['cloud', 'net']
kind_prop['description'] = "* `cloud` - Cloud\\n* `net` - Network"
# Filter PatchedCredentialTypeRequest (PATCH) - field is optional
if 'PatchedCredentialTypeRequest' in schemas:
kind_prop = schemas['PatchedCredentialTypeRequest'].get('properties', {}).get('kind', {})
if 'enum' in kind_prop:
# Filter to only cloud and net (None allowed - field can be omitted in PATCH)
kind_prop['enum'] = ['cloud', 'net', None]
kind_prop['description'] = "* `cloud` - Cloud\\n* `net` - Network"
return result
class CustomAutoSchema(AutoSchema):
"""Custom AutoSchema to add swagger_topic to tags and handle deprecated endpoints."""

View File

@@ -122,6 +122,7 @@ from awx.main.scheduler.task_manager_models import TaskManagerModels
from awx.main.redact import UriCleaner, REPLACE_STR
from awx.main.signals import update_inventory_computed_fields
from awx.main.validators import vars_validate_or_raise
from awx.api.versioning import reverse
@@ -174,8 +175,8 @@ SUMMARIZABLE_FK_FIELDS = {
'workflow_approval': DEFAULT_SUMMARY_FIELDS + ('timeout',),
'schedule': DEFAULT_SUMMARY_FIELDS + ('next_run',),
'unified_job_template': DEFAULT_SUMMARY_FIELDS + ('unified_job_type',),
# last_job and last_job_host_summary are derived from JobHostSummary in HostSerializer,
# not from the stale FK fields on Host.
'last_job': DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'license_error', 'canceled_on'),
'last_job_host_summary': DEFAULT_SUMMARY_FIELDS + ('failed',),
'last_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
'current_update': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
'current_job': DEFAULT_SUMMARY_FIELDS + ('status', 'failed', 'license_error'),
@@ -961,32 +962,14 @@ class UnifiedJobSerializer(BaseSerializer):
class UnifiedJobListSerializer(UnifiedJobSerializer):
# these fields can be included optionally in the response
OPTIONAL_INCLUDE_FIELDS = frozenset({'artifacts', 'extra_vars'})
# these fields are stripped from the response
_STRIPPED_FIELDS = frozenset({'job_args', 'job_cwd', 'job_env', 'result_traceback', 'event_processing_finished', 'artifacts', 'extra_vars'})
class Meta:
fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback', '-event_processing_finished', '-artifacts', '-extra_vars')
# processes the include query param if present
def _requested_includes(self):
request = self.context.get('request')
if request is None:
return frozenset()
raw = request.query_params.get('include', '')
requested = {name.strip() for name in raw.split(',') if name.strip()}
# only allow the fields listed in OPTIONAL_INCLUDE_FIELDS
return frozenset(requested) & self.OPTIONAL_INCLUDE_FIELDS
fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback', '-event_processing_finished', '-artifacts')
def get_field_names(self, declared_fields, info):
field_names = super(UnifiedJobListSerializer, self).get_field_names(declared_fields, info)
# Meta multiple inheritance and -field_name options don't seem to be
# taking effect above, so remove the undesired fields here.
strip = self._STRIPPED_FIELDS - self._requested_includes()
return tuple(x for x in field_names if x not in strip)
return tuple(x for x in field_names if x not in ('job_args', 'job_cwd', 'job_env', 'result_traceback', 'event_processing_finished', 'artifacts'))
def get_types(self):
if type(self) is UnifiedJobListSerializer:
@@ -1039,7 +1022,7 @@ class UnifiedJobStdoutSerializer(UnifiedJobSerializer):
class UserSerializer(BaseSerializer):
password = serializers.CharField(required=False, default='', allow_blank=True, help_text=_('Field used to change the password.'))
password = serializers.CharField(required=False, default='', help_text=_('Field used to change the password.'))
is_system_auditor = serializers.BooleanField(default=False)
show_capabilities = ['edit', 'delete']
@@ -1247,7 +1230,7 @@ class OrganizationSerializer(BaseSerializer, OpaQueryPathMixin):
# to a team. This provides a hint to the ui so it can know to not
# display these roles for team role selection.
for key in ('admin_role', 'member_role'):
if summary_dict and key in summary_dict.get('object_roles', {}):
if key in summary_dict.get('object_roles', {}):
summary_dict['object_roles'][key]['user_only'] = True
return summary_dict
@@ -1855,35 +1838,19 @@ class HostSerializer(BaseSerializerWithVariables):
res['ansible_facts'] = self.reverse('api:host_ansible_facts_detail', kwargs={'pk': obj.instance_id})
if obj.inventory:
res['inventory'] = self.reverse('api:inventory_detail', kwargs={'pk': obj.inventory.pk})
last_summary = obj.latest_summary
if last_summary:
res['last_job_host_summary'] = self.reverse('api:job_host_summary_detail', kwargs={'pk': last_summary.pk})
if last_summary.job_id:
res['last_job'] = self.reverse('api:job_detail', kwargs={'pk': last_summary.job_id})
if obj.last_job:
res['last_job'] = self.reverse('api:job_detail', kwargs={'pk': obj.last_job.pk})
if obj.last_job_host_summary:
res['last_job_host_summary'] = self.reverse('api:job_host_summary_detail', kwargs={'pk': obj.last_job_host_summary.pk})
return res
def get_summary_fields(self, obj):
d = super(HostSerializer, self).get_summary_fields(obj)
last_summary = obj.latest_summary
if last_summary:
d['last_job_host_summary'] = OrderedDict()
d['last_job_host_summary']['id'] = last_summary.id
d['last_job_host_summary']['failed'] = last_summary.failed
try:
last_job = last_summary.job
d['last_job'] = OrderedDict()
for field in DEFAULT_SUMMARY_FIELDS + ('finished', 'status', 'failed', 'canceled_on'):
fval = getattr(last_job, field, None)
if fval is not None:
d['last_job'][field] = fval
if last_job.job_template:
d['last_job']['job_template_id'] = last_job.job_template.id
d['last_job']['job_template_name'] = last_job.job_template.name
except ObjectDoesNotExist:
pass
else:
d.pop('last_job', None)
d.pop('last_job_host_summary', None)
try:
d['last_job']['job_template_id'] = obj.last_job.job_template.id
d['last_job']['job_template_name'] = obj.last_job.job_template.name
except (KeyError, AttributeError):
pass
if has_model_field_prefetched(obj, 'groups'):
group_list = sorted([{'id': g.id, 'name': g.name} for g in obj.groups.all()], key=lambda x: x['id'])[:5]
else:
@@ -1958,16 +1925,14 @@ class HostSerializer(BaseSerializerWithVariables):
return ret
if 'inventory' in ret and not obj.inventory:
ret['inventory'] = None
last_summary = obj.latest_summary
if 'last_job' in ret:
ret['last_job'] = last_summary.job_id if last_summary else None
if 'last_job_host_summary' in ret:
ret['last_job_host_summary'] = last_summary.pk if last_summary else None
if 'last_job' in ret and not obj.last_job:
ret['last_job'] = None
if 'last_job_host_summary' in ret and not obj.last_job_host_summary:
ret['last_job_host_summary'] = None
return ret
def get_has_active_failures(self, obj):
last_summary = obj.latest_summary
return bool(last_summary and last_summary.failed)
return bool(obj.last_job_host_summary and obj.last_job_host_summary.failed)
def get_has_inventory_sources(self, obj):
return obj.inventory_sources.exists()
@@ -2114,17 +2079,9 @@ class BulkHostCreateSerializer(serializers.Serializer):
if request and not request.user.is_superuser:
if request.user not in inv.admin_role:
raise serializers.ValidationError(_(f'Inventory with id {inv.id} not found or lack permissions to add hosts.'))
# Performance optimization (AAP-67978): Instead of loading ALL host names from
# the inventory, only check if the specific new names already exist in the database.
current_hostnames = set(inv.hosts.values_list('name', flat=True))
new_names = [host['name'] for host in attrs['hosts']]
new_name_counts = Counter(new_names)
duplicates_in_new = [name for name, count in new_name_counts.items() if count > 1]
unique_new_names = list(new_name_counts.keys())
existing_duplicates = list(Host.objects.filter(inventory=inv, name__in=unique_new_names).values_list('name', flat=True))
duplicate_new_names = list(set(duplicates_in_new + existing_duplicates))
duplicate_new_names = [n for n in new_names if n in current_hostnames or new_names.count(n) > 1]
if duplicate_new_names:
raise serializers.ValidationError(_(f'Hostnames must be unique in an inventory. Duplicates found: {duplicate_new_names}'))
@@ -2208,13 +2165,13 @@ class BulkHostDeleteSerializer(serializers.Serializer):
attrs['hosts_data'] = attrs['host_qs'].values()
if len(attrs['host_qs']) == 0:
error_hosts = dict.fromkeys(attrs['hosts'], "Hosts do not exist or you lack permission to delete it")
error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in attrs['hosts']}
raise serializers.ValidationError({'hosts': error_hosts})
if len(attrs['host_qs']) < len(attrs['hosts']):
hosts_exists = [host['id'] for host in attrs['hosts_data']]
failed_hosts = list(set(attrs['hosts']).difference(hosts_exists))
error_hosts = dict.fromkeys(failed_hosts, "Hosts do not exist or you lack permission to delete it")
error_hosts = {host: "Hosts do not exist or you lack permission to delete it" for host in failed_hosts}
raise serializers.ValidationError({'hosts': error_hosts})
# Getting all inventories that the hosts can be in
@@ -2975,19 +2932,6 @@ class CredentialTypeSerializer(BaseSerializer):
field['label'] = _(field['label'])
if 'help_text' in field:
field['help_text'] = _(field['help_text'])
# Deep copy inputs to avoid modifying the original model data
inputs = value.get('inputs')
if not isinstance(inputs, dict):
inputs = {}
value['inputs'] = copy.deepcopy(inputs)
fields = value['inputs'].get('fields', [])
if not isinstance(fields, list):
fields = []
# Normalize fields and filter out internal fields
value['inputs']['fields'] = [f for f in fields if not f.get('internal')]
return value
def filter_field_metadata(self, fields, method):
@@ -3583,7 +3527,7 @@ class JobRelaunchSerializer(BaseSerializer):
choices=NEW_JOB_TYPE_CHOICES,
write_only=True,
)
credential_passwords = VerbatimField(required=False, write_only=True)
credential_passwords = VerbatimField(required=True, write_only=True)
class Meta:
model = Job
@@ -4178,28 +4122,9 @@ class LaunchConfigurationBaseSerializer(BaseSerializer):
attrs['extra_data'][key] = db_extra_data[key]
# Build unsaved version of this config, use it to detect prompts errors
# Capture keys before _build_mock_obj pops pseudo-fields from attrs
incoming_attr_keys = set(attrs.keys())
mock_obj = self._build_mock_obj(attrs)
ask_mapping_keys = set(ujt.get_ask_mapping().keys())
requested_prompt_fields = incoming_attr_keys & ask_mapping_keys
if 'extra_data' in incoming_attr_keys:
requested_prompt_fields.add('extra_vars')
requested_prompt_fields.add('survey_passwords')
# prompts_dict() pulls persisted M2M state (labels, credentials,
# instance_groups) via the instance pk. Only re-validate the full prompt
# state when the caller is switching the underlying template; otherwise
# restrict validation to the fields the request explicitly provided.
if 'unified_job_template' in attrs:
prompts_to_validate = mock_obj.prompts_dict()
elif requested_prompt_fields:
prompts_to_validate = {k: v for k, v in mock_obj.prompts_dict().items() if k in requested_prompt_fields}
else:
prompts_to_validate = None
if prompts_to_validate is not None:
accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(_exclude_errors=self.exclude_errors, **prompts_to_validate)
if set(list(ujt.get_ask_mapping().keys()) + ['extra_data']) & set(attrs.keys()):
accepted, rejected, errors = ujt._accept_or_ignore_job_kwargs(_exclude_errors=self.exclude_errors, **mock_obj.prompts_dict())
else:
# Only perform validation of prompts if prompts fields are provided
errors = {}

View File

@@ -1,6 +1,6 @@
{% if content_only %}<div class="nocode ansi_fore ansi_back{% if dark %} ansi_dark{% endif %}">{% else %}
<!DOCTYPE HTML>
<html lang="en">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>{{ title }}</title>

View File

@@ -1,4 +1,4 @@
---
collections:
- name: ansible.receptor
version: 2.0.8
version: 2.0.6

View File

@@ -14,14 +14,13 @@ import sys
import time
from base64 import b64encode
from collections import OrderedDict
from jwt import decode as _jwt_decode
from urllib3.exceptions import ConnectTimeoutError
# Django
from django.conf import settings
from django.core.exceptions import FieldError, ObjectDoesNotExist
from django.db.models import Q, Sum, Count, Subquery, OuterRef
from django.db.models import Q, Sum, Count
from django.db import IntegrityError, ProgrammingError, transaction, connection
from django.db.models.fields.related import ManyToManyField, ForeignKey
from django.db.models.functions import Trunc
@@ -53,19 +52,13 @@ from ansi2html import Ansi2HTMLConverter
from datetime import timezone as dt_timezone
from wsgiref.util import FileWrapper
from drf_spectacular.utils import extend_schema_view, extend_schema
# django-ansible-base
from ansible_base.lib.utils.requests import get_remote_hosts
from ansible_base.rbac.models import RoleEvaluation
from ansible_base.lib.utils.schema import extend_schema_if_available
from ansible_base.lib.workload_identity.controller import AutomationControllerJobScope
# flags
from flags.state import flag_enabled
# AWX
from awx.main.utils.workload_identity import retrieve_workload_identity_jwt_with_claims
from awx.main.tasks.system import send_notifications, update_inventory_computed_fields
from awx.main.access import get_user_queryset
from awx.api.generics import (
@@ -127,7 +120,6 @@ from awx.api.views.mixin import (
RelatedJobsPreventDeleteMixin,
UnifiedJobDeletionMixin,
NoTruncateMixin,
UnifiedJobIncludeMixin,
)
from awx.api.pagination import UnifiedJobEventPagination
from awx.main.utils import set_environ
@@ -210,12 +202,11 @@ class DashboardView(APIView):
groups_inventory_failed = models.Group.objects.filter(inventory_sources__last_job_failed=True).count()
data['groups'] = {'url': reverse('api:group_list', request=request), 'total': user_groups.count(), 'inventory_failed': groups_inventory_failed}
user_hosts = get_user_queryset(request.user, models.Host).exclude(inventory__kind='constructed')
latest_summary_failed = Subquery(models.JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id').values('failed')[:1])
user_hosts_failed = user_hosts.annotate(_latest_failed=latest_summary_failed).filter(_latest_failed=True)
user_hosts = get_user_queryset(request.user, models.Host)
user_hosts_failed = user_hosts.filter(last_job_host_summary__failed=True)
data['hosts'] = {
'url': reverse('api:host_list', request=request),
'failures_url': reverse('api:host_list', request=request) + "?last_job_host_summary__failed=True",
'total': user_hosts.count(),
'failed': user_hosts_failed.count(),
}
@@ -387,10 +378,6 @@ class DashboardJobsGraphView(APIView):
class InstanceList(ListCreateAPIView):
"""
Creates an instance if used on a Kubernetes or OpenShift deployment of Ansible Automation Platform.
"""
name = _("Instances")
model = models.Instance
serializer_class = serializers.InstanceSerializer
@@ -802,11 +789,22 @@ class TeamRolesList(SubListAttachDetachAPIView):
data = dict(msg=_("You cannot grant system-level permissions to a team."))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
if not request.data.get('disassociate'):
team = get_object_or_404(models.Team, pk=self.kwargs['pk'])
content_object = role.content_object
if hasattr(content_object, 'validate_role_assignment'):
content_object.validate_role_assignment(team, role_definition=None, requesting_user=request.user)
team = get_object_or_404(models.Team, pk=self.kwargs['pk'])
credential_content_type = ContentType.objects.get_for_model(models.Credential)
if role.content_type == credential_content_type:
if not role.content_object.organization:
data = dict(
msg=_("You cannot grant access to a credential that is not assigned to an organization (private credentials cannot be assigned to teams)")
)
return Response(data, status=status.HTTP_400_BAD_REQUEST)
elif role.content_object.organization.id != team.organization.id:
if not request.user.is_superuser:
data = dict(
msg=_(
"You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
)
)
return Response(data, status=status.HTTP_400_BAD_REQUEST)
return super(TeamRolesList, self).post(request, *args, **kwargs)
@@ -1265,12 +1263,19 @@ class UserRolesList(SubListAttachDetachAPIView):
if not sub_id:
return super(UserRolesList, self).post(request)
if not request.data.get('disassociate'):
role = get_object_or_400(models.Role, pk=sub_id)
user = get_object_or_400(models.User, pk=self.kwargs['pk'])
content_object = role.content_object
if hasattr(content_object, 'validate_role_assignment'):
content_object.validate_role_assignment(user, role_definition=None, requesting_user=request.user)
user = get_object_or_400(models.User, pk=self.kwargs['pk'])
role = get_object_or_400(models.Role, pk=sub_id)
content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential) # dict of {model: content_type}
credential_content_type = content_types[models.Credential]
if role.content_type == credential_content_type:
if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
if not role.content_object.organization and not request.user.is_superuser:
data = dict(msg=_("You cannot grant private credential access to another user"))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
return super(UserRolesList, self).post(request, *args, **kwargs)
@@ -1449,7 +1454,7 @@ class CredentialList(ListCreateAPIView):
@extend_schema_if_available(
extensions={
"x-ai-description": "Create a new credential. The `inputs` field contain type-specific input fields. The required fields depend on related `credential_type`. Use GET /v2/credential_types/{id}/ (tool name: controller.credential_types_retrieve) and inspect `inputs` field for the specific credential type's expected schema. The fields `user` and `team` are deprecated and should not be included in the payload."
"x-ai-description": "Create a new credential. The `inputs` field contain type-specific input fields. The required fields depend on related `credential_type`. Use GET /v2/credential_types/{id}/ (tool name: controller.credential_types_retrieve) and inspect `inputs` field for the specific credential type's expected schema."
}
)
def post(self, request, *args, **kwargs):
@@ -1585,175 +1590,7 @@ class CredentialCopy(CopyAPIView):
resource_purpose = 'copy of a credential'
class OIDCCredentialTestMixin:
"""
Mixin to add OIDC workload identity token support to credential test endpoints.
This mixin provides methods to handle OIDC-enabled external credentials that use
workload identity tokens for authentication.
"""
@staticmethod
def _get_workload_identity_token(job_template: models.JobTemplate, audience: str) -> str:
"""Generate a workload identity token for a job template.
Args:
job_template: The JobTemplate instance to generate claims for
audience: The JWT audience claim value
Returns:
str: The generated JWT token
"""
claims = {
AutomationControllerJobScope.CLAIM_ORGANIZATION_NAME: job_template.organization.name,
AutomationControllerJobScope.CLAIM_ORGANIZATION_ID: job_template.organization.id,
AutomationControllerJobScope.CLAIM_PROJECT_NAME: job_template.project.name,
AutomationControllerJobScope.CLAIM_PROJECT_ID: job_template.project.id,
AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_NAME: job_template.name,
AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_ID: job_template.id,
AutomationControllerJobScope.CLAIM_PLAYBOOK_NAME: job_template.playbook,
}
return retrieve_workload_identity_jwt_with_claims(
claims=claims,
audience=audience,
scope=AutomationControllerJobScope.name,
)
@staticmethod
def _decode_jwt_payload_for_display(jwt_token):
"""Decode JWT payload for display purposes only (signature not verified).
This is safe because the JWT was just created by AWX and is only decoded
to show the user what claims are being sent to the external system.
The external system will perform proper signature verification.
Args:
jwt_token: The JWT token to decode
Returns:
dict: The decoded JWT payload
"""
return _jwt_decode(jwt_token, algorithms=["RS256"], options={"verify_signature": False}) # NOSONAR python:S5659
def _has_workload_identity_token(self, credential_type_inputs):
"""Check if credential type has an internal workload_identity_token field.
Args:
credential_type_inputs: The inputs dict from a credential type
Returns:
bool: True if the credential type has a workload_identity_token field marked as internal
"""
fields = credential_type_inputs.get('fields', []) if isinstance(credential_type_inputs, dict) else []
return any(field.get('internal') and field.get('id') == 'workload_identity_token' for field in fields)
def _validate_and_get_job_template(self, job_template_id):
"""Validate job template ID and return the JobTemplate instance.
Args:
job_template_id: The job template ID from metadata
Returns:
JobTemplate instance
Raises:
ParseError: If job_template_id is invalid or not found
"""
if job_template_id is None:
raise ParseError(_('Job template ID is required.'))
try:
return models.JobTemplate.objects.get(id=int(job_template_id))
except ValueError:
raise ParseError(_('Job template ID must be an integer.'))
except models.JobTemplate.DoesNotExist:
raise ParseError(_('Job template with ID %(id)s does not exist.') % {'id': job_template_id})
def _handle_oidc_credential_test(self, backend_kwargs):
"""
Handle OIDC workload identity token generation for external credential test endpoints.
This method should only be called when FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED is enabled
and the credential type has a workload_identity_token field.
Args:
backend_kwargs: The kwargs dict to pass to the backend (will be modified in place)
Returns:
dict: Response body containing details with the sent JWT payload
Raises:
PermissionDenied: If user lacks access to the job template (re-raised for 403 response)
All other exceptions are caught and converted to 400 responses with error details.
Modifies backend_kwargs in place to add workload_identity_token.
"""
# Validate job template
job_template_id = backend_kwargs.pop('job_template_id', None)
job_template = self._validate_and_get_job_template(job_template_id)
# Check user access
if not self.request.user.can_access(models.JobTemplate, 'start', job_template):
raise PermissionDenied(_('You do not have access to job template with id: %(id)s.') % {'id': job_template.id})
# Generate workload identity token
jwt_token = self._get_workload_identity_token(job_template, backend_kwargs.get('url'))
backend_kwargs['workload_identity_token'] = jwt_token
return {'details': {'sent_jwt_payload': self._decode_jwt_payload_for_display(jwt_token)}}
def _call_backend_with_error_handling(self, plugin, backend_kwargs, response_body):
"""Call credential backend and handle errors."""
try:
with set_environ(**settings.AWX_TASK_ENV):
plugin.backend(**backend_kwargs)
return Response(response_body, status=status.HTTP_202_ACCEPTED)
except requests.exceptions.HTTPError as exc:
message = self._extract_http_error_message(exc)
self._add_error_to_response(response_body, message)
return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
except Exception as exc:
message = self._extract_generic_error_message(exc)
self._add_error_to_response(response_body, message)
return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
@staticmethod
def _extract_http_error_message(exc):
"""Extract error message from HTTPError, checking response JSON and text."""
message = str(exc)
if not hasattr(exc, 'response') or exc.response is None:
return message
try:
error_data = exc.response.json()
if 'errors' in error_data and error_data['errors']:
return ', '.join(error_data['errors'])
if 'error' in error_data:
return error_data['error']
except (ValueError, KeyError):
if exc.response.text:
return exc.response.text
return message
@staticmethod
def _extract_generic_error_message(exc):
"""Extract error message from exception, handling ConnectTimeoutError specially."""
message = str(exc) if str(exc) else exc.__class__.__name__
for arg in getattr(exc, 'args', []):
if isinstance(getattr(arg, 'reason', None), ConnectTimeoutError):
return str(arg.reason)
return message
@staticmethod
def _add_error_to_response(response_body, message):
"""Add error message to both 'detail' and 'details.error_message' fields."""
response_body['detail'] = message
if 'details' in response_body:
response_body['details']['error_message'] = message
class CredentialExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
class CredentialExternalTest(SubDetailAPIView):
"""
Test updates to the input values and metadata of an external credential
before saving them.
@@ -1766,15 +1603,9 @@ class CredentialExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
obj_permission_type = 'use'
resource_purpose = 'test external credential'
@extend_schema_if_available(extensions={"x-ai-description": """Test update the input values and metadata of an external credential.
This endpoint supports testing credentials that connect to external secret management systems
such as CyberArk AIM, CyberArk Conjur, HashiCorp Vault, AWS Secrets Manager, Azure Key Vault,
Centrify Vault, Thycotic DevOps Secrets Vault, and GitHub App Installation Access Token Lookup.
It does not support standard credential types such as Machine, SCM, and Cloud."""})
@extend_schema_if_available(extensions={"x-ai-description": "Test update the input values and metadata of an external credential"})
def post(self, request, *args, **kwargs):
obj = self.get_object()
if obj.credential_type.kind != 'external':
raise ParseError(_('Credential is not testable.'))
backend_kwargs = {}
for field_name, value in obj.inputs.items():
backend_kwargs[field_name] = obj.get_input(field_name)
@@ -1782,22 +1613,20 @@ class CredentialExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
if value != '$encrypted$':
backend_kwargs[field_name] = value
backend_kwargs.update(request.data.get('metadata', {}))
# Handle OIDC workload identity token generation if enabled
response_body = {}
if flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED') and self._has_workload_identity_token(obj.credential_type.inputs):
try:
oidc_response_body = self._handle_oidc_credential_test(backend_kwargs)
response_body.update(oidc_response_body)
except PermissionDenied:
raise
except Exception as exc:
error_message = str(exc.detail) if hasattr(exc, 'detail') else str(exc)
response_body['detail'] = error_message
response_body['details'] = {'error_message': error_message}
return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
return self._call_backend_with_error_handling(obj.credential_type.plugin, backend_kwargs, response_body)
try:
with set_environ(**settings.AWX_TASK_ENV):
obj.credential_type.plugin.backend(**backend_kwargs)
return Response({}, status=status.HTTP_202_ACCEPTED)
except requests.exceptions.HTTPError as exc:
message = 'HTTP {}'.format(exc.response.status_code)
return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
except Exception as exc:
message = exc.__class__.__name__
args = getattr(exc, 'args', [])
for a in args:
if isinstance(getattr(a, 'reason', None), ConnectTimeoutError):
message = str(a.reason)
return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
class CredentialInputSourceDetail(RetrieveUpdateDestroyAPIView):
@@ -1827,7 +1656,7 @@ class CredentialInputSourceSubList(SubListCreateAPIView):
parent_key = 'target_credential'
class CredentialTypeExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
class CredentialTypeExternalTest(SubDetailAPIView):
"""
Test a complete set of input values for an external credential before
saving it.
@@ -1842,26 +1671,21 @@ class CredentialTypeExternalTest(OIDCCredentialTestMixin, SubDetailAPIView):
@extend_schema_if_available(extensions={"x-ai-description": "Test a complete set of input values for an external credential"})
def post(self, request, *args, **kwargs):
obj = self.get_object()
if obj.kind != 'external':
raise ParseError(_('Credential type is not testable.'))
backend_kwargs = request.data.get('inputs', {})
backend_kwargs.update(request.data.get('metadata', {}))
# Handle OIDC workload identity token generation if enabled
response_body = {}
if flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED') and self._has_workload_identity_token(obj.inputs):
try:
oidc_response_body = self._handle_oidc_credential_test(backend_kwargs)
response_body.update(oidc_response_body)
except PermissionDenied:
raise
except Exception as exc:
error_message = str(exc.detail) if hasattr(exc, 'detail') else str(exc)
response_body['detail'] = error_message
response_body['details'] = {'error_message': error_message}
return Response(response_body, status=status.HTTP_400_BAD_REQUEST)
return self._call_backend_with_error_handling(obj.plugin, backend_kwargs, response_body)
try:
obj.plugin.backend(**backend_kwargs)
return Response({}, status=status.HTTP_202_ACCEPTED)
except requests.exceptions.HTTPError as exc:
message = 'HTTP {}'.format(exc.response.status_code)
return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
except Exception as exc:
message = exc.__class__.__name__
args = getattr(exc, 'args', [])
for a in args:
if isinstance(getattr(a, 'reason', None), ConnectTimeoutError):
message = str(a.reason)
return Response({'inputs': message}, status=status.HTTP_400_BAD_REQUEST)
class HostRelatedSearchMixin(object):
@@ -1927,7 +1751,7 @@ class HostList(HostRelatedSearchMixin, ListCreateAPIView):
if filter_string:
filter_qs = SmartFilter.query_from_string(filter_string)
qs &= filter_qs
return qs.distinct().with_latest_summary_id()
return qs.distinct()
def list(self, *args, **kwargs):
try:
@@ -1942,9 +1766,6 @@ class HostDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIView):
serializer_class = serializers.HostSerializer
resource_purpose = 'host detail'
def get_queryset(self):
return super().get_queryset().with_latest_summary_id()
@extend_schema_if_available(extensions={"x-ai-description": "Delete a host"})
def delete(self, request, *args, **kwargs):
if self.get_object().inventory.pending_deletion:
@@ -1978,9 +1799,6 @@ class InventoryHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIVie
filter_read_permission = False
resource_purpose = 'hosts of an inventory'
def get_queryset(self):
return super().get_queryset().with_latest_summary_id()
class HostGroupsList(SubListCreateAttachDetachAPIView):
'''the list of groups a host is directly a member of'''
@@ -2164,9 +1982,6 @@ class GroupHostsList(HostRelatedSearchMixin, SubListCreateAttachDetachAPIView):
relationship = 'hosts'
resource_purpose = 'hosts of a group'
def get_queryset(self):
return super().get_queryset().with_latest_summary_id()
def update_raw_data(self, data):
data.pop('inventory', None)
return super(GroupHostsList, self).update_raw_data(data)
@@ -2198,7 +2013,7 @@ class GroupAllHostsList(HostRelatedSearchMixin, SubListAPIView):
self.check_parent_access(parent)
qs = self.request.user.get_queryset(self.model).distinct() # need distinct for '&' operator
sublist_qs = parent.all_hosts.distinct()
return (qs & sublist_qs).with_latest_summary_id()
return qs & sublist_qs
class GroupInventorySourcesList(SubListAPIView):
@@ -2491,9 +2306,6 @@ class InventorySourceHostsList(HostRelatedSearchMixin, SubListDestroyAPIView):
check_sub_obj_permission = False
resource_purpose = 'hosts of an inventory source'
def get_queryset(self):
return super().get_queryset().with_latest_summary_id()
def perform_list_destroy(self, instance_list):
inv_source = self.get_parent_object()
with ignore_inventory_computed_fields():
@@ -2657,11 +2469,6 @@ class JobTemplateDetail(RelatedJobsPreventDeleteMixin, RetrieveUpdateDestroyAPIV
resource_purpose = 'job template detail'
@extend_schema_view(
retrieve=extend_schema(
extensions={'x-ai-description': 'List job template launch criteria'},
)
)
class JobTemplateLaunch(RetrieveAPIView):
model = models.JobTemplate
obj_permission_type = 'start'
@@ -2670,9 +2477,6 @@ class JobTemplateLaunch(RetrieveAPIView):
resource_purpose = 'launch a job from a job template'
def update_raw_data(self, data):
"""
Use the ID of a job template to retrieve its launch details.
"""
try:
obj = self.get_object()
except PermissionDenied:
@@ -3506,11 +3310,6 @@ class WorkflowJobTemplateLabelList(JobTemplateLabelList):
resource_purpose = 'labels of a workflow job template'
@extend_schema_view(
retrieve=extend_schema(
extensions={'x-ai-description': 'List workflow job template launch criteria.'},
)
)
class WorkflowJobTemplateLaunch(RetrieveAPIView):
model = models.WorkflowJobTemplate
obj_permission_type = 'start'
@@ -3519,9 +3318,6 @@ class WorkflowJobTemplateLaunch(RetrieveAPIView):
resource_purpose = 'launch a workflow job from a workflow job template'
def update_raw_data(self, data):
"""
Use the ID of a workflow job template to retrieve its launch details.
"""
try:
obj = self.get_object()
except PermissionDenied:
@@ -3851,7 +3647,7 @@ class SystemJobTemplateNotificationTemplatesSuccessList(SystemJobTemplateNotific
resource_purpose = 'notification templates triggered on system job success'
class JobList(UnifiedJobIncludeMixin, ListAPIView):
class JobList(ListAPIView):
model = models.Job
serializer_class = serializers.JobListSerializer
resource_purpose = 'jobs'
@@ -3914,11 +3710,6 @@ class JobCancel(GenericCancelView):
return super().post(request, *args, **kwargs)
@extend_schema_view(
retrieve=extend_schema(
extensions={'x-ai-description': 'List job relaunch criteria'},
)
)
class JobRelaunch(RetrieveAPIView):
model = models.Job
obj_permission_type = 'start'
@@ -3926,7 +3717,6 @@ class JobRelaunch(RetrieveAPIView):
resource_purpose = 'relaunch a job'
def update_raw_data(self, data):
"""Use the ID of a job to retrieve data on retry attempts and necessary passwords."""
data = super(JobRelaunch, self).update_raw_data(data)
try:
obj = self.get_object()
@@ -4568,7 +4358,7 @@ class UnifiedJobTemplateList(ListAPIView):
resource_purpose = 'unified job templates'
class UnifiedJobList(UnifiedJobIncludeMixin, ListAPIView):
class UnifiedJobList(ListAPIView):
model = models.UnifiedJob
serializer_class = serializers.UnifiedJobListSerializer
search_fields = ('description', 'name', 'job__playbook')
@@ -4871,12 +4661,19 @@ class RoleUsersList(SubListAttachDetachAPIView):
if not sub_id:
return super(RoleUsersList, self).post(request)
if not request.data.get('disassociate'):
user = get_object_or_400(models.User, pk=sub_id)
role = self.get_parent_object()
content_object = role.content_object
if hasattr(content_object, 'validate_role_assignment'):
content_object.validate_role_assignment(user, role_definition=None, requesting_user=request.user)
user = get_object_or_400(models.User, pk=sub_id)
role = self.get_parent_object()
content_types = ContentType.objects.get_for_models(models.Organization, models.Team, models.Credential) # dict of {model: content_type}
credential_content_type = content_types[models.Credential]
if role.content_type == credential_content_type:
if 'disassociate' not in request.data and role.content_object.organization and user not in role.content_object.organization.member_role:
data = dict(msg=_("You cannot grant credential access to a user not in the credentials' organization"))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
if not role.content_object.organization and not request.user.is_superuser:
data = dict(msg=_("You cannot grant private credential access to another user"))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
return super(RoleUsersList, self).post(request, *args, **kwargs)
@@ -4909,6 +4706,24 @@ class RoleTeamsList(SubListAttachDetachAPIView):
data = dict(msg=_("You cannot assign an Organization participation role as a child role for a Team."))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
credential_content_type = ContentType.objects.get_for_model(models.Credential)
if role.content_type == credential_content_type:
# Private credentials (no organization) are never allowed for teams
if not role.content_object.organization:
data = dict(
msg=_("You cannot grant access to a credential that is not assigned to an organization (private credentials cannot be assigned to teams)")
)
return Response(data, status=status.HTTP_400_BAD_REQUEST)
# Cross-organization credentials are only allowed for superusers
elif role.content_object.organization.id != team.organization.id:
if not request.user.is_superuser:
data = dict(
msg=_(
"You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
)
)
return Response(data, status=status.HTTP_400_BAD_REQUEST)
action = 'attach'
if request.data.get('disassociate', None):
action = 'unattach'
@@ -4917,11 +4732,6 @@ class RoleTeamsList(SubListAttachDetachAPIView):
data = dict(msg=_("You cannot grant system-level permissions to a team."))
return Response(data, status=status.HTTP_400_BAD_REQUEST)
if action == 'attach':
content_object = role.content_object
if hasattr(content_object, 'validate_role_assignment'):
content_object.validate_role_assignment(team, role_definition=None, requesting_user=request.user)
if not request.user.can_access(self.parent_model, action, role, team, self.relationship, request.data, skip_sub_obj_read_check=False):
raise PermissionDenied()
if request.data.get('disassociate', None):

View File

@@ -9,7 +9,7 @@ from django.utils import translation
from awx.api.generics import APIView, Response
from awx.api.permissions import AnalyticsPermission
from awx.api.versioning import reverse
from awx.main.utils import get_awx_version, set_environ
from awx.main.utils import get_awx_version
from awx.main.utils.analytics_proxy import OIDCClient
from rest_framework import status
@@ -49,6 +49,7 @@ class GetNotAllowedMixin(object):
class AnalyticsRootView(APIView):
permission_classes = (AnalyticsPermission,)
name = _('Automation Analytics')
swagger_topic = 'Automation Analytics'
resource_purpose = 'automation analytics endpoints'
@extend_schema_if_available(extensions={"x-ai-description": "A list of additional API endpoints related to analytics"})
@@ -210,32 +211,31 @@ class AnalyticsGenericView(APIView):
return self._error_response(ERROR_UNSUPPORTED_METHOD, method, remote=False, status_code=status.HTTP_500_INTERNAL_SERVER_ERROR)
url = self._get_analytics_url(request.path)
using_subscriptions_credentials = False
with set_environ(**settings.AWX_TASK_ENV):
try:
rh_user = getattr(settings, 'REDHAT_USERNAME', None)
rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
if not (rh_user and rh_password):
rh_user = self._get_setting('SUBSCRIPTIONS_CLIENT_ID', None, ERROR_MISSING_USER)
rh_password = self._get_setting('SUBSCRIPTIONS_CLIENT_SECRET', None, ERROR_MISSING_PASSWORD)
using_subscriptions_credentials = True
try:
rh_user = getattr(settings, 'REDHAT_USERNAME', None)
rh_password = getattr(settings, 'REDHAT_PASSWORD', None)
if not (rh_user and rh_password):
rh_user = self._get_setting('SUBSCRIPTIONS_CLIENT_ID', None, ERROR_MISSING_USER)
rh_password = self._get_setting('SUBSCRIPTIONS_CLIENT_SECRET', None, ERROR_MISSING_PASSWORD)
using_subscriptions_credentials = True
client = OIDCClient(rh_user, rh_password)
response = client.make_request(
method,
url,
headers=headers,
verify=settings.INSIGHTS_CERT_PATH,
params=getattr(request, 'query_params', {}),
json=getattr(request, 'data', {}),
timeout=(31, 31),
)
except requests.RequestException:
# subscriptions credentials are not valid for basic auth, so just return 401
if using_subscriptions_credentials:
response = Response(status=status.HTTP_401_UNAUTHORIZED)
else:
logger.error("Automation Analytics API request failed, trying base auth method")
response = self._base_auth_request(request, method, url, rh_user, rh_password, headers)
client = OIDCClient(rh_user, rh_password)
response = client.make_request(
method,
url,
headers=headers,
verify=settings.INSIGHTS_CERT_PATH,
params=getattr(request, 'query_params', {}),
json=getattr(request, 'data', {}),
timeout=(31, 31),
)
except requests.RequestException:
# subscriptions credentials are not valid for basic auth, so just return 401
if using_subscriptions_credentials:
response = Response(status=status.HTTP_401_UNAUTHORIZED)
else:
logger.error("Automation Analytics API request failed, trying base auth method")
response = self._base_auth_request(request, method, url, rh_user, rh_password, headers)
#
# Missing or wrong user/pass
#
@@ -306,6 +306,7 @@ class AnalyticsAuthorizedView(AnalyticsGenericListView):
class AnalyticsReportsList(GetNotAllowedMixin, AnalyticsGenericListView):
name = _("Reports")
swagger_topic = "Automation Analytics"
resource_purpose = 'automation analytics reports'

View File

@@ -212,9 +212,3 @@ class NoTruncateMixin(object):
if self.request.query_params.get('no_truncate'):
context.update(no_truncate=True)
return context
class UnifiedJobIncludeMixin(object):
# Reserve the name 'include' so we can use it as a query param. Otherwise, the rest-filters backend
# would treat it as a model field lookup.
rest_filters_reserved_names = ('include',)

View File

@@ -25,6 +25,7 @@ import requests
from ansible_base.lib.utils.schema import extend_schema_if_available
from awx import MODE
from awx.api.generics import APIView
from awx.conf.registry import settings_registry
from awx.main.analytics import all_collectors
@@ -32,7 +33,7 @@ from awx.main.ha import is_ha_environment
from awx.main.tasks.system import clear_setting_cache
from awx.main.utils import get_awx_version, get_custom_venv_choices
from awx.main.utils.licensing import validate_entitlement_manifest
from awx.api.versioning import URLPathVersioning, reverse
from awx.api.versioning import URLPathVersioning, reverse, drf_reverse
from awx.main.constants import PRIVILEGE_ESCALATION_METHODS
from awx.main.models import Project, Organization, Instance, InstanceGroup, JobTemplate
from awx.main.utils import set_environ
@@ -61,6 +62,8 @@ class ApiRootView(APIView):
data['custom_logo'] = settings.CUSTOM_LOGO
data['custom_login_info'] = settings.CUSTOM_LOGIN_INFO
data['login_redirect_override'] = settings.LOGIN_REDIRECT_OVERRIDE
if MODE == 'development':
data['docs'] = drf_reverse('api:schema-swagger-ui')
return Response(data)
@@ -344,22 +347,13 @@ class ApiV2ConfigView(APIView):
become_methods=PRIVILEGE_ESCALATION_METHODS,
)
# Check superuser/auditor first
if request.user.is_superuser or request.user.is_system_auditor:
has_org_access = True
else:
# Single query checking all three organization role types at once
has_org_access = (
(
Organization.access_qs(request.user, 'change')
| Organization.access_qs(request.user, 'audit')
| Organization.access_qs(request.user, 'add_project')
)
.distinct()
.exists()
)
if has_org_access:
if (
request.user.is_superuser
or request.user.is_system_auditor
or Organization.accessible_objects(request.user, 'admin_role').exists()
or Organization.accessible_objects(request.user, 'auditor_role').exists()
or Organization.accessible_objects(request.user, 'project_admin_role').exists()
):
data.update(
dict(
project_base_dir=settings.PROJECTS_ROOT,
@@ -367,10 +361,8 @@ class ApiV2ConfigView(APIView):
custom_virtualenvs=get_custom_venv_choices(),
)
)
else:
# Only check JobTemplate access if org check failed
if JobTemplate.accessible_objects(request.user, 'admin_role').exists():
data['custom_virtualenvs'] = get_custom_venv_choices()
elif JobTemplate.accessible_objects(request.user, 'admin_role').exists():
data['custom_virtualenvs'] = get_custom_venv_choices()
return Response(data)

View File

@@ -17,7 +17,7 @@ from awx.api import serializers
from awx.api.generics import APIView, GenericAPIView
from awx.api.permissions import WebhookKeyPermission
from awx.main.models import Job, JobTemplate, WorkflowJob, WorkflowJobTemplate
from awx.main.utils.common import get_job_variable_prefixes
from awx.main.constants import JOB_VARIABLE_PREFIXES
logger = logging.getLogger('awx.api.views.webhooks')
@@ -133,7 +133,7 @@ class WebhookReceiverBase(APIView):
@csrf_exempt
@extend_schema_if_available(extensions={"x-ai-description": "Receive a webhook event and trigger a job"})
def post(self, request, *args, **kwargs_in):
def post(self, request, *args, **kwargs):
# Ensure that the full contents of the request are captured for multiple uses.
request.body
@@ -166,7 +166,7 @@ class WebhookReceiverBase(APIView):
'extra_vars': {},
}
for name in get_job_variable_prefixes():
for name in JOB_VARIABLE_PREFIXES:
kwargs['extra_vars']['{}_webhook_event_type'.format(name)] = event_type
kwargs['extra_vars']['{}_webhook_event_guid'.format(name)] = event_guid
kwargs['extra_vars']['{}_webhook_event_ref'.format(name)] = event_ref

View File

@@ -897,6 +897,8 @@ class HostAccess(BaseAccess):
'created_by',
'modified_by',
'inventory',
'last_job__job_template',
'last_job_host_summary__job',
)
prefetch_related = ('groups', 'inventory_sources')

View File

@@ -8,7 +8,6 @@ import pathlib
import shutil
import tarfile
import tempfile
from urllib.parse import urlparse, urlunparse
from django.conf import settings
from django.core.serializers.json import DjangoJSONEncoder
@@ -24,8 +23,6 @@ from awx.main.models import Job
from awx.main.access import access_registry
from awx.main.utils import get_awx_http_client_headers, set_environ, datetime_hook
from awx.main.utils.analytics_proxy import OIDCClient
from awx.main.utils.candlepin import get_or_generate_candlepin_certificate
from awx.main.utils.candlepin.client import _temp_cert_files
__all__ = ['register', 'gather', 'ship']
@@ -44,76 +41,6 @@ def _valid_license():
return True
def _get_cert_upload_url(url):
"""
Convert analytics URL to use 'cert.' subdomain for mTLS uploads.
Some analytics services use different hostnames for different auth methods:
- cert.example.com - for mTLS (certificate-based) uploads
- example.com - for OIDC (token-based) uploads
Args:
url: Original analytics URL
Returns:
URL with 'cert.' prepended to hostname if not already present
"""
try:
parsed = urlparse(url)
hostname = parsed.hostname
# Only modify if hostname doesn't already start with 'cert.'
if hostname and not hostname.startswith('cert.'):
new_hostname = f'cert.{hostname}'
# Reconstruct URL with new hostname
netloc = new_hostname
if parsed.port:
netloc = f'{new_hostname}:{parsed.port}'
new_parsed = parsed._replace(netloc=netloc)
return urlunparse(new_parsed)
return url
except Exception as e:
logger.warning(f'Could not modify URL for cert upload: {e}, using original URL')
return url
def _get_analytics_credentials():
"""
Get Red Hat Insights credentials from settings.
Attempts to retrieve credentials in the following priority order:
1. REDHAT_USERNAME / REDHAT_PASSWORD
2. SUBSCRIPTIONS_USERNAME / SUBSCRIPTIONS_PASSWORD
3. SUBSCRIPTIONS_CLIENT_ID / SUBSCRIPTIONS_CLIENT_SECRET
Returns:
tuple: (username, password) if credentials are found, (None, None) otherwise
"""
rh_id = getattr(settings, 'REDHAT_USERNAME', None)
rh_secret = getattr(settings, 'REDHAT_PASSWORD', None)
if rh_id and rh_secret:
return rh_id, rh_secret
# Try SUBSCRIPTIONS_USERNAME / SUBSCRIPTIONS_PASSWORD
rh_id = getattr(settings, 'SUBSCRIPTIONS_USERNAME', None)
rh_secret = getattr(settings, 'SUBSCRIPTIONS_PASSWORD', None)
if rh_id and rh_secret:
return rh_id, rh_secret
# Try SUBSCRIPTIONS_CLIENT_ID / SUBSCRIPTIONS_CLIENT_SECRET
rh_id = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
rh_secret = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
if rh_id and rh_secret:
return rh_id, rh_secret
return None, None
def all_collectors():
from awx.main.analytics import collectors
@@ -257,8 +184,10 @@ def gather(dest=None, module=None, subset=None, since=None, until=None, collecti
logger.log(log_level, "Automation Analytics not enabled. Use --dry-run to gather locally without sending.")
return None
rh_id, rh_secret = _get_analytics_credentials()
if not (settings.AUTOMATION_ANALYTICS_URL and rh_id and rh_secret):
if not (
settings.AUTOMATION_ANALYTICS_URL
and ((settings.REDHAT_USERNAME and settings.REDHAT_PASSWORD) or (settings.SUBSCRIPTIONS_CLIENT_ID and settings.SUBSCRIPTIONS_CLIENT_SECRET))
):
logger.log(log_level, "Not gathering analytics, configuration is invalid. Use --dry-run to gather locally without sending.")
return None
@@ -439,14 +368,19 @@ def ship(path):
logger.error('AUTOMATION_ANALYTICS_URL is not set')
return False
rh_id, rh_secret = _get_analytics_credentials()
rh_id = getattr(settings, 'REDHAT_USERNAME', None)
rh_secret = getattr(settings, 'REDHAT_PASSWORD', None)
if not (rh_id and rh_secret):
rh_id = getattr(settings, 'SUBSCRIPTIONS_CLIENT_ID', None)
rh_secret = getattr(settings, 'SUBSCRIPTIONS_CLIENT_SECRET', None)
if not rh_id:
logger.error('No valid username found. Tried: REDHAT_USERNAME, SUBSCRIPTIONS_USERNAME, SUBSCRIPTIONS_CLIENT_ID')
logger.error('Neither REDHAT_USERNAME nor SUBSCRIPTIONS_CLIENT_ID are set')
return False
if not rh_secret:
logger.error('No valid password found. Tried: REDHAT_PASSWORD, SUBSCRIPTIONS_PASSWORD, SUBSCRIPTIONS_CLIENT_SECRET')
logger.error('Neither REDHAT_PASSWORD nor SUBSCRIPTIONS_CLIENT_SECRET are set')
return False
with open(path, 'rb') as f:
@@ -454,40 +388,17 @@ def ship(path):
s = requests.Session()
s.headers = get_awx_http_client_headers()
s.headers.pop('Content-Type')
with set_environ(**settings.AWX_TASK_ENV):
# Try Certificate-based mTLS authentication (zero-touch)
cert_pem, key_pem = get_or_generate_candlepin_certificate()
if cert_pem and key_pem:
# Use cert. subdomain for mTLS uploads
cert_url = _get_cert_upload_url(url)
logger.debug("Attempting certificate-based authentication for analytics upload")
try:
with _temp_cert_files(cert_pem, key_pem) as (cert_path, key_path):
response = s.post(
cert_url, files=files, cert=(cert_path, key_path), verify=settings.INSIGHTS_CERT_PATH, headers=s.headers, timeout=(31, 31)
)
if response.status_code < 300:
return True
else:
logger.warning(
f'Certificate-based authentication failed with status {response.status_code}, {response.text}. Falling back to OIDC auth'
)
except Exception as e:
logger.warning(f"Certificate-based authentication failed: {e}, falling back to OIDC auth")
# Try OIDC authentication
logger.debug("Attempting OIDC authentication for analytics upload")
f.seek(0) # requests POST may read from the handler, so seek to beginning of file for the next POST attempt
try:
client = OIDCClient(rh_id, rh_secret)
response = client.make_request("POST", url, headers=s.headers, files=files, verify=settings.INSIGHTS_CERT_PATH, timeout=(31, 31))
except requests.RequestException:
logger.error("Automation Analytics API request failed, trying base auth method")
response = s.post(url, files=files, verify=settings.INSIGHTS_CERT_PATH, auth=(rh_id, rh_secret), headers=s.headers, timeout=(31, 31))
if response.status_code < 300:
return True
else:
logger.error(f'OIDC authentication failed with status {response.status_code}, {response.text}')
return False
except requests.RequestException as e:
logger.error(f"OIDC authentication failed: {e}")
return False
# Accept 2XX status_codes
if response.status_code >= 300:
logger.error('Upload failed with status {}, {}'.format(response.status_code, response.text))
return False
return True

View File

@@ -1,41 +0,0 @@
import http.client
import socket
import urllib.error
import urllib.request
import logging
from django.conf import settings
logger = logging.getLogger(__name__)
def get_dispatcherd_metrics(request):
metrics_cfg = settings.METRICS_SUBSYSTEM_CONFIG.get('server', {}).get(settings.METRICS_SERVICE_DISPATCHER, {})
host = metrics_cfg.get('host', 'localhost')
port = metrics_cfg.get('port', 8015)
metrics_filter = []
if request is not None and hasattr(request, "query_params"):
try:
nodes_filter = request.query_params.getlist("node")
except Exception:
nodes_filter = []
if nodes_filter and settings.CLUSTER_HOST_ID not in nodes_filter:
return ''
try:
metrics_filter = request.query_params.getlist("metric")
except Exception:
metrics_filter = []
if metrics_filter:
# Right now we have no way of filtering the dispatcherd metrics
# so just avoid getting in the way if another metric is filtered for
return ''
url = f"http://{host}:{port}/metrics"
try:
with urllib.request.urlopen(url, timeout=1.0) as response:
payload = response.read()
if not payload:
return ''
return payload.decode('utf-8')
except (urllib.error.URLError, UnicodeError, socket.timeout, TimeoutError, http.client.HTTPException) as exc:
logger.debug(f"Failed to collect dispatcherd metrics from {url}: {exc}")
return ''

View File

@@ -15,7 +15,6 @@ from rest_framework.request import Request
from awx.main.consumers import emit_channel_notification
from awx.main.utils import is_testing
from awx.main.utils.redis import get_redis_client
from .dispatcherd_metrics import get_dispatcherd_metrics
root_key = settings.SUBSYSTEM_METRICS_REDIS_KEY_PREFIX
logger = logging.getLogger('awx.main.analytics')
@@ -399,6 +398,11 @@ class DispatcherMetrics(Metrics):
SetFloatM('workflow_manager_recorded_timestamp', 'Unix timestamp when metrics were last recorded'),
SetFloatM('workflow_manager_spawn_workflow_graph_jobs_seconds', 'Time spent spawning workflow tasks'),
SetFloatM('workflow_manager_get_tasks_seconds', 'Time spent loading workflow tasks from db'),
# dispatcher subsystem metrics
SetIntM('dispatcher_pool_scale_up_events', 'Number of times local dispatcher scaled up a worker since startup'),
SetIntM('dispatcher_pool_active_task_count', 'Number of active tasks in the worker pool when last task was submitted'),
SetIntM('dispatcher_pool_max_worker_count', 'Highest number of workers in worker pool in last collection interval, about 20s'),
SetFloatM('dispatcher_availability', 'Fraction of time (in last collection interval) dispatcher was able to receive messages'),
]
def __init__(self, *args, **kwargs):
@@ -426,12 +430,8 @@ class CallbackReceiverMetrics(Metrics):
def metrics(request):
output_text = ''
output_text += DispatcherMetrics().generate_metrics(request)
output_text += CallbackReceiverMetrics().generate_metrics(request)
dispatcherd_metrics = get_dispatcherd_metrics(request)
if dispatcherd_metrics:
output_text += dispatcherd_metrics
for m in [DispatcherMetrics(), CallbackReceiverMetrics()]:
output_text += m.generate_metrics(request)
return output_text
@@ -481,6 +481,13 @@ class CallbackReceiverMetricsServer(MetricsServer):
super().__init__(settings.METRICS_SERVICE_CALLBACK_RECEIVER, registry)
class DispatcherMetricsServer(MetricsServer):
def __init__(self):
registry = CollectorRegistry(auto_describe=True)
registry.register(CustomToPrometheusMetricsCollector(DispatcherMetrics(metrics_have_changed=False)))
super().__init__(settings.METRICS_SERVICE_DISPATCHER, registry)
class WebsocketsMetricsServer(MetricsServer):
def __init__(self):
registry = CollectorRegistry(auto_describe=True)

View File

@@ -82,7 +82,7 @@ class MainConfig(AppConfig):
def configure_dispatcherd(self):
"""This implements the default configuration for dispatcherd
If running the tasking service like awx-manage dispatcherd,
If running the tasking service like awx-manage run_dispatcher,
some additional config will be applied on top of this.
This configuration provides the minimum such that code can submit
tasks to pg_notify to run those tasks.

View File

@@ -213,40 +213,6 @@ register(
category_slug='system',
)
register(
'AWX_ANALYTICS_CANDLEPIN_CA',
field_class=fields.CharField,
default='/etc/rhsm/ca/redhat-uep.pem',
allow_blank=True,
label=_('Candlepin CA Certificate Path'),
help_text=_('Path to the CA certificate file for verifying TLS connections to Candlepin. Leave blank to use system certificates.'),
category=_('System'),
category_slug='system',
)
register(
'AWX_ANALYTICS_CANDLEPIN_RENEWAL_THRESHOLD_DAYS',
field_class=fields.IntegerField,
default=90,
min_value=1,
label=_('Candlepin Certificate Renewal Threshold'),
help_text=_('Number of days before certificate expiry to trigger automatic renewal of Candlepin identity certificates.'),
category=_('System'),
category_slug='system',
unit=_('days'),
)
register(
'AWX_ANALYTICS_CANDLEPIN_PROXY_URL',
field_class=fields.CharField,
default='',
allow_blank=True,
label=_('Candlepin Proxy URL'),
help_text=_('HTTP/HTTPS proxy URL for Candlepin API requests (e.g., http://proxy.example.com:8080). Leave blank for no proxy.'),
category=_('System'),
category_slug='system',
)
register(
'INSTALL_UUID',
field_class=fields.CharField,
@@ -325,22 +291,6 @@ register(
category_slug='jobs',
)
register(
'INCLUDE_DEPRECATED_AWX_VAR_PREFIX',
field_class=fields.BooleanField,
default=True,
label=_('Include Deprecated AWX Variable Prefix'),
help_text=_(
'When enabled (default), auto-generated job variables are emitted '
'with both the tower_ prefix and the deprecated awx_ prefix for '
'backward compatibility. Disable to emit only tower_ prefixed '
'variables and eliminate duplicates. The awx_ prefix is deprecated '
'and this setting will default to False in a future release.'
),
category=_('Jobs'),
category_slug='jobs',
)
register(
'AWX_ISOLATION_BASE_PATH',
field_class=fields.CharField,
@@ -874,58 +824,6 @@ register(
unit=_('seconds'),
)
register(
'CANDLEPIN_CONSUMER_UUID',
field_class=fields.CharField,
default='',
allow_blank=True,
encrypted=False,
label=_('Candlepin Consumer UUID'),
help_text=_('UUID of the registered Candlepin consumer for this AAP instance.'),
category=_('System'),
category_slug='system',
hidden=True,
)
register(
'CANDLEPIN_CERT_PEM',
field_class=fields.CharField,
default='',
allow_blank=True,
encrypted=True,
label=_('Candlepin Identity Certificate'),
help_text=_('PEM-encoded Candlepin identity certificate for mTLS authentication.'),
category=_('System'),
category_slug='system',
hidden=True,
)
register(
'CANDLEPIN_KEY_PEM',
field_class=fields.CharField,
default='',
allow_blank=True,
encrypted=True,
label=_('Candlepin Identity Key'),
help_text=_('PEM-encoded private key for Candlepin identity certificate.'),
category=_('System'),
category_slug='system',
hidden=True,
)
register(
'CANDLEPIN_SERIAL_NUMBER',
field_class=fields.CharField,
default='',
allow_blank=True,
encrypted=False,
label=_('Candlepin Certificate Serial Number'),
help_text=_('Serial number of the Candlepin identity certificate for tracking.'),
category=_('System'),
category_slug='system',
hidden=True,
)
register(
'IS_K8S',
field_class=fields.BooleanField,

View File

@@ -11,7 +11,6 @@ __all__ = [
'CAN_CANCEL',
'ACTIVE_STATES',
'STANDARD_INVENTORY_UPDATE_ENV',
'OIDC_CREDENTIAL_TYPE_NAMESPACES',
]
PRIVILEGE_ESCALATION_METHODS = [
@@ -100,6 +99,10 @@ MAX_ISOLATED_PATH_COLON_DELIMITER = 2
SURVEY_TYPE_MAPPING = {'text': str, 'textarea': str, 'password': str, 'multiplechoice': str, 'multiselect': str, 'integer': int, 'float': (float, int)}
JOB_VARIABLE_PREFIXES = [
'awx',
'tower',
]
# Note, the \u001b[... are ansi color codes. We don't currenly import any of the python modules which define the codes.
# Importing a library just for this message seemed like overkill
@@ -137,6 +140,3 @@ org_role_to_permission = {
'execution_environment_admin_role': 'add_executionenvironment',
'auditor_role': 'view_project', # TODO: also doesnt really work
}
# OIDC credential type namespaces for feature flag filtering
OIDC_CREDENTIAL_TYPE_NAMESPACES = ['hashivault-kv-oidc', 'hashivault-ssh-oidc']

View File

@@ -27,15 +27,10 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
"pool_kwargs": {
"min_workers": settings.JOB_EVENT_WORKERS,
"max_workers": max_workers,
# This must be less than max_workers to make sense, which is usually 4
# With reserve of 1, after a burst of tasks, load needs to down to 4-1=3
# before we return to min_workers
"scaledown_reserve": 1,
"worker_max_lifetime_seconds": settings.WORKER_MAX_LIFETIME_SECONDS,
},
"main_kwargs": {"node_id": settings.CLUSTER_HOST_ID},
"process_manager_cls": "ForkServerManager",
"process_manager_kwargs": {"preload_modules": ['awx.main.dispatch.prefork']},
"process_manager_kwargs": {"preload_modules": ['awx.main.dispatch.hazmat']},
},
"brokers": {},
"publish": {},
@@ -43,8 +38,8 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
}
if mock_publish:
config["brokers"]["dispatcherd.testing.brokers.noop"] = {}
config["publish"]["default_broker"] = "dispatcherd.testing.brokers.noop"
config["brokers"]["noop"] = {}
config["publish"]["default_broker"] = "noop"
else:
config["brokers"]["pg_notify"] = {
"config": get_pg_notify_params(),
@@ -61,11 +56,5 @@ def get_dispatcherd_config(for_service: bool = False, mock_publish: bool = False
}
config["brokers"]["pg_notify"]["channels"] = ['tower_broadcast_all', 'tower_settings_change', get_task_queuename()]
metrics_cfg = settings.METRICS_SUBSYSTEM_CONFIG.get('server', {}).get(settings.METRICS_SERVICE_DISPATCHER)
if metrics_cfg:
config["service"]["metrics_kwargs"] = {
"host": metrics_cfg.get("host", "localhost"),
"port": metrics_cfg.get("port", 8015),
}
return config

View File

@@ -18,7 +18,7 @@ django.setup() # noqa
from django.conf import settings
# Preload all periodic tasks so their imports will be in shared memory
for name, options in settings.DISPATCHER_SCHEDULE.items():
for name, options in settings.CELERYBEAT_SCHEDULE.items():
resolve_callable(options['task'])

View File

@@ -1,4 +1,6 @@
import logging
import os
import time
from multiprocessing import Process
@@ -13,12 +15,13 @@ class PoolWorker(object):
"""
A simple wrapper around a multiprocessing.Process that tracks a worker child process.
The worker process runs the provided target function.
The worker process runs the provided target function and tracks its creation time.
"""
def __init__(self, target, args):
def __init__(self, target, args, **kwargs):
self.process = Process(target=target, args=args)
self.process.daemon = True
self.creation_time = time.monotonic()
def start(self):
self.process.start()
@@ -35,20 +38,44 @@ class WorkerPool(object):
pool = WorkerPool(workers_num=4) # spawn four worker processes
"""
def __init__(self, workers_num=None):
self.workers_num = workers_num or settings.JOB_EVENT_WORKERS
pool_cls = PoolWorker
debug_meta = ''
def init_workers(self, target):
def __init__(self, workers_num=None):
self.name = settings.CLUSTER_HOST_ID
self.pid = os.getpid()
self.workers_num = workers_num or settings.JOB_EVENT_WORKERS
self.workers = []
def __len__(self):
return len(self.workers)
def init_workers(self, target, *target_args):
self.target = target
self.target_args = target_args
for idx in range(self.workers_num):
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and cache connections (that way lies race conditions)
django_connection.close()
django_cache.close()
worker = PoolWorker(target, (idx,))
try:
worker.start()
except Exception:
logger.exception('could not fork')
else:
logger.debug('scaling up worker pid:{}'.format(worker.process.pid))
self.up()
def up(self):
idx = len(self.workers)
# It's important to close these because we're _about_ to fork, and we
# don't want the forked processes to inherit the open sockets
# for the DB and cache connections (that way lies race conditions)
django_connection.close()
django_cache.close()
worker = self.pool_cls(self.target, (idx,) + self.target_args)
self.workers.append(worker)
try:
worker.start()
except Exception:
logger.exception('could not fork')
else:
logger.debug('scaling up worker pid:{}'.format(worker.process.pid))
return idx, worker
def stop(self, signum):
try:
for worker in self.workers:
os.kill(worker.pid, signum)
except Exception:
logger.exception('could not kill {}'.format(worker.pid))

View File

@@ -1,6 +1,9 @@
from datetime import timedelta
import logging
from django.db.models import Q
from django.conf import settings
from django.utils.timezone import now as tz_now
from django.contrib.contenttypes.models import ContentType
from awx.main.models import Instance, UnifiedJob, WorkflowJob
@@ -47,6 +50,26 @@ def reap_job(j, status, job_explanation=None):
logger.error(f'{j.log_format} is no longer {status_before}; reaping')
def reap_waiting(instance=None, status='failed', job_explanation=None, grace_period=None, excluded_uuids=None, ref_time=None):
"""
Reap all jobs in waiting for this instance.
"""
if grace_period is None:
grace_period = settings.JOB_WAITING_GRACE_PERIOD + settings.TASK_MANAGER_TIMEOUT
if instance is None:
hostname = Instance.objects.my_hostname()
else:
hostname = instance.hostname
if ref_time is None:
ref_time = tz_now()
jobs = UnifiedJob.objects.filter(status='waiting', modified__lte=ref_time - timedelta(seconds=grace_period), controller_node=hostname)
if excluded_uuids:
jobs = jobs.exclude(celery_task_id__in=excluded_uuids)
for j in jobs:
reap_job(j, status, job_explanation=job_explanation)
def reap(instance=None, status='failed', job_explanation=None, excluded_uuids=None, ref_time=None):
"""
Reap all jobs in running for this instance.

View File

@@ -19,24 +19,49 @@ def signame(sig):
return dict((k, v) for v, k in signal.__dict__.items() if v.startswith('SIG') and not v.startswith('SIG_'))[sig]
class AWXConsumerRedis(object):
class WorkerSignalHandler:
def __init__(self):
self.kill_now = False
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, self.exit_gracefully)
def exit_gracefully(self, *args, **kwargs):
self.kill_now = True
class AWXConsumerBase(object):
last_stats = time.time()
def __init__(self, name, worker, queues=[], pool=None):
self.should_stop = False
def __init__(self, name, worker):
self.name = name
self.pool = WorkerPool()
self.pool.init_workers(worker.work_loop)
self.total_messages = 0
self.queues = queues
self.worker = worker
self.pool = pool
if pool is None:
self.pool = WorkerPool()
self.pool.init_workers(self.worker.work_loop)
self.redis = get_redis_client()
def run(self):
def run(self, *args, **kwargs):
signal.signal(signal.SIGINT, self.stop)
signal.signal(signal.SIGTERM, self.stop)
# Child should implement other things here
def stop(self, signum, frame):
self.should_stop = True
logger.warning('received {}, stopping'.format(signame(signum)))
raise SystemExit()
class AWXConsumerRedis(AWXConsumerBase):
def run(self, *args, **kwargs):
super(AWXConsumerRedis, self).run(*args, **kwargs)
logger.info(f'Callback receiver started with pid={os.getpid()}')
db.connection.close() # logs use database, so close connection
while True:
time.sleep(60)
def stop(self, signum, frame):
logger.warning('received {}, stopping'.format(signame(signum)))
raise SystemExit()

View File

@@ -26,6 +26,7 @@ from awx.main.models.events import emit_event_detail
from awx.main.utils.profiling import AWXProfiler
from awx.main.tasks.system import events_processed_hook
import awx.main.analytics.subsystem_metrics as s_metrics
from .base import WorkerSignalHandler
logger = logging.getLogger('awx.main.commands.run_callback_receiver')
@@ -56,16 +57,6 @@ def job_stats_wrapup(job_identifier, event=None):
logger.exception('Worker failed to save stats or emit notifications: Job {}'.format(job_identifier))
class WorkerSignalHandler:
def __init__(self):
self.kill_now = False
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, self.exit_gracefully)
def exit_gracefully(self, *args, **kwargs):
self.kill_now = True
class CallbackBrokerWorker:
"""
A worker implementation that deserializes callback event data and persists
@@ -77,13 +68,13 @@ class CallbackBrokerWorker:
MAX_RETRIES = 2
INDIVIDUAL_EVENT_RETRIES = 3
last_stats = time.time()
last_flush = time.time()
total = 0
last_event = ''
prof = None
def __init__(self):
self.last_stats = time.time()
self.last_flush = time.time()
self.buff = {}
self.redis = get_redis_client()
self.subsystem_metrics = s_metrics.CallbackReceiverMetrics(auto_pipe_execute=False)

View File

@@ -1,3 +1,4 @@
import inspect
import logging
import importlib
import time
@@ -36,13 +37,18 @@ def run_callable(body):
if 'guid' in body:
set_guid(body.pop('guid'))
_call = resolve_callable(task)
if inspect.isclass(_call):
# the callable is a class, e.g., RunJob; instantiate and
# return its `run()` method
_call = _call().run
log_extra = ''
logger_method = logger.debug
if 'time_pub' in body:
time_publish = time.time() - body['time_pub']
if time_publish > 5.0:
if ('time_ack' in body) and ('time_pub' in body):
time_publish = body['time_ack'] - body['time_pub']
time_waiting = time.time() - body['time_ack']
if time_waiting > 5.0 or time_publish > 5.0:
# If task too a very long time to process, add this information to the log
log_extra = f' took {time_publish:.4f} to send message'
log_extra = f' took {time_publish:.4f} to ack, {time_waiting:.4f} in local dispatcher'
logger_method = logger.info
# don't print kwargs, they often contain launch-time secrets
logger_method(f'task {uuid} starting {task}(*{args}){log_extra}')

View File

@@ -428,9 +428,6 @@ class CredentialInputField(JSONSchemaField):
# determine the defined fields for the associated credential type
properties = {}
for field in model_instance.credential_type.inputs.get('fields', []):
# Prevent users from providing values for internally resolved fields
if 'internal' in field:
continue
field = field.copy()
properties[field['id']] = field
if field.get('choices', []):
@@ -569,7 +566,6 @@ class CredentialTypeInputField(JSONSchemaField):
},
'label': {'type': 'string'},
'help_text': {'type': 'string'},
'internal': {'type': 'boolean'},
'multiline': {'type': 'boolean'},
'secret': {'type': 'boolean'},
'ask_at_runtime': {'type': 'boolean'},

View File

@@ -1,330 +0,0 @@
import sys
from argparse import RawDescriptionHelpFormatter
from django.core.management.base import BaseCommand
from awx.main.utils.candlepin.client import CandlepinClient
from awx.main.utils.candlepin.lifecycle import (
get_candlepin_ca,
get_candlepin_url,
get_proxy_url,
get_renewal_days,
needs_renewal,
parse_cert,
)
from awx.main.utils.candlepin import (
_fetch_candlepin_cert_from_db,
_save_candlepin_cert_to_db,
_save_candlepin_registration_to_db,
resolve_registration_credentials,
)
class Command(BaseCommand):
"""
Manage Candlepin consumer registration and certificate lifecycle.
Subcommands:
register Register this AAP instance as a Candlepin consumer and obtain an
identity certificate for mTLS analytics uploads.
renew Perform a manual check-in and, if needed, renew the stored identity
certificate.
"""
help = 'Manage Candlepin consumer registration and certificate lifecycle'
def create_parser(self, prog_name, subcommand, **kwargs):
return super().create_parser(
prog_name,
subcommand,
formatter_class=RawDescriptionHelpFormatter,
epilog='\n'.join(
[
'SUBCOMMANDS',
'',
' register Register this instance as a Candlepin consumer.',
' Credentials are read from AWX database by default',
' (REDHAT_USERNAME, REDHAT_PASSWORD). The organization is',
' discovered automatically from the Candlepin account.',
' Pass --username / --password-stdin / --org to override.',
' Example: echo "password" | awx-manage candlepin_cert register --username user --password-stdin',
'',
' renew Perform a manual check-in and proactive cert renewal.',
' Reads the stored cert/key/UUID from database.',
' Use --force to renew even if the cert is not near expiry.',
'',
'CONFIGURATION',
'',
' Settings can be configured via Django settings (awx/settings/defaults.py):',
'',
' AWX_ANALYTICS_CANDLEPIN_URL Candlepin base URL',
' (default: https://subscription.example.com/candlepin)',
' AWX_ANALYTICS_CANDLEPIN_CA Path to Candlepin CA cert for TLS verification',
' AWX_ANALYTICS_CANDLEPIN_RENEWAL_THRESHOLD_DAYS Days before expiry to trigger renewal (default: 90)',
' AWX_ANALYTICS_CANDLEPIN_PROXY_URL HTTP/HTTPS proxy for Candlepin API calls',
]
),
**kwargs,
)
def add_arguments(self, parser):
subparsers = parser.add_subparsers(dest='subcommand', metavar='subcommand')
subparsers.required = True
# --- register ---
reg = subparsers.add_parser(
'register',
help='Register this instance as a Candlepin consumer',
formatter_class=RawDescriptionHelpFormatter,
)
reg.add_argument('--username', help='Red Hat subscription username (overrides REDHAT_USERNAME from database)')
reg.add_argument(
'--password-stdin', dest='password_stdin', action='store_true', help='Read password from stdin (overrides REDHAT_PASSWORD from database)'
)
reg.add_argument('--org', help='Candlepin owner/org key (overrides auto-discovered organization)')
reg.add_argument('--candlepin-url', dest='candlepin_url', help='Candlepin base URL (overrides AWX_ANALYTICS_CANDLEPIN_URL setting)')
reg.add_argument(
'--candlepin-ca', dest='candlepin_ca', help='Path to Candlepin CA cert for TLS verification (overrides AWX_ANALYTICS_CANDLEPIN_CA setting)'
)
reg.add_argument('--proxy', help='HTTP/HTTPS proxy URL (overrides AWX_ANALYTICS_CANDLEPIN_PROXY_URL setting)')
reg.add_argument('--no-verify-tls', dest='no_verify_tls', action='store_true', help='Disable TLS certificate verification for Candlepin API calls')
reg.add_argument('--force', action='store_true', help='Re-register even if a certificate already exists in database')
reg.add_argument('--dry-run', dest='dry_run', action='store_true', help='Perform registration but do not save the result to database')
# --- renew ---
ren = subparsers.add_parser(
'renew',
help='Check in and renew the Candlepin identity certificate',
formatter_class=RawDescriptionHelpFormatter,
)
ren.add_argument('--candlepin-url', dest='candlepin_url', help='Candlepin base URL (overrides AWX_ANALYTICS_CANDLEPIN_URL setting)')
ren.add_argument(
'--candlepin-ca', dest='candlepin_ca', help='Path to Candlepin CA cert for TLS verification (overrides AWX_ANALYTICS_CANDLEPIN_CA setting)'
)
ren.add_argument('--proxy', help='HTTP/HTTPS proxy URL (overrides AWX_ANALYTICS_CANDLEPIN_PROXY_URL setting)')
ren.add_argument('--no-verify-tls', dest='no_verify_tls', action='store_true', help='Disable TLS certificate verification for Candlepin API calls')
ren.add_argument('--force', action='store_true', help='Renew the certificate even if it is not near expiry')
ren.add_argument('--dry-run', dest='dry_run', action='store_true', help='Perform check-in and renewal but do not save the result to database')
def handle(self, *args, **options):
subcommand = options['subcommand']
if subcommand == 'register':
ok = self._handle_register(options)
elif subcommand == 'renew':
ok = self._handle_renew(options)
else:
self.stderr.write(f'Unknown subcommand: {subcommand}')
sys.exit(1)
if not ok:
sys.exit(1)
# ------------------------------------------------------------------
# register
# ------------------------------------------------------------------
def _resolve_and_validate_credentials(self, options):
"""Merge CLI options with DB values and validate all required fields are present.
Returns ``(username, password, org, db_install_uuid)`` on success, or ``None``
if any required field is missing (errors are written to ``self.stderr``).
"""
username_override = options.get('username')
org_override = options.get('org')
verify_tls = not options.get('no_verify_tls', False)
# Read password from stdin if --password-stdin is set
if options.get('password_stdin'):
password_override = sys.stdin.read().strip()
if not password_override:
self.stderr.write('--password-stdin specified but no password provided on stdin')
return None
else:
password_override = None
# Use shared resolution and validation function
username, password, org, install_uuid, errors = resolve_registration_credentials(
username_override=username_override, password_override=password_override, org_override=org_override, verify_tls=verify_tls
)
if errors:
for error in errors:
self.stderr.write(f'Missing required value: {error}')
return None
return username, password, org, install_uuid
def _handle_register(self, options):
dry_run = options['dry_run']
force = options['force']
# Check whether a cert is already stored unless --force.
existing_cert, existing_key, _ = _fetch_candlepin_cert_from_db()
if existing_cert and existing_key and not force:
self.stdout.write('A Candlepin identity certificate is already stored in database. Use --force to re-register and replace it.')
return True
# Resolve credentials: CLI flags take precedence over database.
resolved = self._resolve_and_validate_credentials(options)
if resolved is None:
return False
username, password, org, db_install_uuid = resolved
candlepin_url = options.get('candlepin_url') or get_candlepin_url()
candlepin_ca = options.get('candlepin_ca') or get_candlepin_ca()
proxy = options.get('proxy') or get_proxy_url()
verify_tls = not options.get('no_verify_tls', False)
# If dry-run, display what would happen and exit early before any Candlepin operations
if dry_run:
self.stdout.write('[dry-run] Would register with Candlepin:')
self.stdout.write(f' URL : {candlepin_url}')
self.stdout.write(f' Organization : {org}')
self.stdout.write(f' Username : {username}')
self.stdout.write(f' Install UUID : {db_install_uuid}')
if candlepin_ca:
self.stdout.write(f' CA cert : {candlepin_ca}')
if proxy:
self.stdout.write(f' Proxy : {proxy}')
self.stdout.write(f' Verify TLS : {verify_tls}')
self.stdout.write('[dry-run] No Candlepin operations performed.')
return True
client = CandlepinClient(base_url=candlepin_url, candlepin_ca=candlepin_ca, proxy=proxy, verify_tls=verify_tls)
self.stdout.write(f'Registering with Candlepin at {candlepin_url} (org={org}) ...')
try:
cert_pem, key_pem, consumer_uuid = client.register_consumer(username, password, org, install_uuid=db_install_uuid)
except Exception as e:
self.stderr.write(f'Registration failed: {e}')
return False
self.stdout.write('Registered successfully.')
self.stdout.write(f' Consumer UUID : {consumer_uuid}')
# Save to database
if _save_candlepin_registration_to_db(cert_pem, key_pem, consumer_uuid):
self.stdout.write('Certificate, key, and consumer UUID saved to database.')
else:
self.stderr.write('Failed to save registration to database.')
return False
# Best-effort certificate metadata display
try:
info = parse_cert(cert_pem)
self.stdout.write(f' Cert serial : {info["serial"]}')
self.stdout.write(f' Cert CN : {info["cn"]}')
self.stdout.write(f' Valid until : {info["not_after"]} ({info["days_remaining"]} days remaining)')
except ValueError as e:
self.stdout.write(f'Certificate metadata unavailable: {e}')
return True
# ------------------------------------------------------------------
# renew
# ------------------------------------------------------------------
def _handle_renew(self, options):
dry_run = options['dry_run']
force = options['force']
cert_pem, key_pem, consumer_uuid = _fetch_candlepin_cert_from_db()
if not cert_pem or not key_pem:
self.stderr.write('No Candlepin identity certificate found in database. Run the register subcommand first.')
return False
if not consumer_uuid:
self.stderr.write('CANDLEPIN_CONSUMER_UUID is not set. Run the register subcommand first.')
return False
try:
info = parse_cert(cert_pem)
self.stdout.write('Current certificate:')
self.stdout.write(f' Serial : {info["serial"]}')
self.stdout.write(f' CN : {info["cn"]}')
self.stdout.write(f' Valid until : {info["not_after"]} ({info["days_remaining"]} days remaining)')
except ValueError as e:
self.stdout.write('Current certificate:')
self.stdout.write(f' Certificate metadata unavailable: {e}')
info = None
candlepin_url = options.get('candlepin_url') or get_candlepin_url()
candlepin_ca = options.get('candlepin_ca') or get_candlepin_ca()
proxy = options.get('proxy') or get_proxy_url()
verify_tls = not options.get('no_verify_tls', False)
renewal_days = get_renewal_days()
# Check if renewal is needed (without force, just check cert expiry locally)
renewal_needed = force or needs_renewal(cert_pem, renewal_days)
# If dry-run, display what would happen and exit early before any Candlepin operations
if dry_run:
self.stdout.write('[dry-run] Would perform the following operations:')
self.stdout.write(f' URL : {candlepin_url}')
self.stdout.write(f' Consumer UUID : {consumer_uuid}')
if candlepin_ca:
self.stdout.write(f' CA cert : {candlepin_ca}')
if proxy:
self.stdout.write(f' Proxy : {proxy}')
self.stdout.write(f' Verify TLS : {verify_tls}')
self.stdout.write(' 1. Check in with Candlepin')
if renewal_needed:
reason = 'forced via --force' if force else f'expiry within {renewal_days} days'
self.stdout.write(f' 2. Renew certificate ({reason})')
else:
if info:
self.stdout.write(f' 2. No renewal needed ({info["days_remaining"]} days remaining, threshold: {renewal_days} days)')
else:
self.stdout.write(f' 2. No renewal needed (threshold: {renewal_days} days)')
self.stdout.write('[dry-run] No Candlepin operations performed.')
return True
client = CandlepinClient(base_url=candlepin_url, candlepin_ca=candlepin_ca, proxy=proxy, verify_tls=verify_tls)
self.stdout.write(f'Checking in with Candlepin at {candlepin_url} (consumer={consumer_uuid}) ...')
checkin_success = client.checkin(consumer_uuid, cert_pem, key_pem)
if not checkin_success:
self.stderr.write('Check-in with Candlepin failed. Unable to verify certificate status.')
self.stderr.write('Certificate renewal may still be needed. Use --force to renew anyway, or check logs for details.')
return False
self.stdout.write('Check-in successful.')
if not renewal_needed:
if info:
self.stdout.write(f'Certificate has {info["days_remaining"]} days remaining (renewal threshold: {renewal_days} days). No renewal needed.')
else:
self.stdout.write(f'Certificate renewal threshold is {renewal_days} days. No renewal needed.')
return True
reason = 'forced via --force' if force else f'expiry within {renewal_days} days'
self.stdout.write(f'Renewing certificate ({reason}) ...')
try:
new_cert_pem, new_key_pem = client.regenerate_cert(consumer_uuid, cert_pem, key_pem)
except Exception as e:
self.stderr.write(f'Certificate renewal failed: {e}')
return False
self.stdout.write('Certificate renewed successfully.')
# Save to database
if _save_candlepin_cert_to_db(new_cert_pem, new_key_pem):
self.stdout.write('Renewed certificate and key saved to database.')
else:
self.stderr.write('Failed to save renewed certificate to database.')
return False
# Best-effort certificate metadata display
try:
new_info = parse_cert(new_cert_pem)
if info:
self.stdout.write(f' Old serial : {info["serial"]}')
self.stdout.write(f' New serial : {new_info["serial"]}')
self.stdout.write(f' Valid until : {new_info["not_after"]} ({new_info["days_remaining"]} days remaining)')
except ValueError as e:
self.stdout.write(f'Certificate metadata unavailable: {e}')
return True

View File

@@ -1,88 +0,0 @@
import argparse
import inspect
import logging
import os
import sys
import yaml
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
from dispatcherd.cli import (
CONTROL_ARG_SCHEMAS,
DEFAULT_CONFIG_FILE,
_base_cli_parent,
_control_common_parent,
_register_control_arguments,
_build_command_data_from_args,
)
from dispatcherd.config import setup as dispatcher_setup
from dispatcherd.factories import get_control_from_settings
from dispatcherd.service import control_tasks
from awx.main.dispatch.config import get_dispatcherd_config
from awx.main.management.commands.dispatcherd import ensure_no_dispatcherd_env_config
logger = logging.getLogger(__name__)
class Command(BaseCommand):
help = 'Dispatcher control operations'
def add_arguments(self, parser):
parser.description = 'Run dispatcherd control commands using awx-manage.'
base_parent = _base_cli_parent()
control_parent = _control_common_parent()
parser._add_container_actions(base_parent)
parser._add_container_actions(control_parent)
subparsers = parser.add_subparsers(dest='command', metavar='command')
subparsers.required = True
shared_parents = [base_parent, control_parent]
for command in control_tasks.__all__:
func = getattr(control_tasks, command, None)
doc = inspect.getdoc(func) or ''
summary = doc.splitlines()[0] if doc else None
command_parser = subparsers.add_parser(
command,
help=summary,
description=doc,
parents=shared_parents,
)
_register_control_arguments(command_parser, CONTROL_ARG_SCHEMAS.get(command))
def handle(self, *args, **options):
command = options.pop('command', None)
if not command:
raise CommandError('No dispatcher control command specified')
for django_opt in ('verbosity', 'traceback', 'no_color', 'force_color', 'skip_checks'):
options.pop(django_opt, None)
log_level = options.pop('log_level', 'DEBUG')
config_path = os.path.abspath(options.pop('config', DEFAULT_CONFIG_FILE))
expected_replies = options.pop('expected_replies', 1)
logging.basicConfig(level=getattr(logging, log_level), stream=sys.stdout)
logger.debug(f"Configured standard out logging at {log_level} level")
default_config = os.path.abspath(DEFAULT_CONFIG_FILE)
ensure_no_dispatcherd_env_config()
if config_path != default_config:
raise CommandError('The config path CLI option is not allowed for the awx-manage command')
if connection.vendor == 'sqlite':
raise CommandError('dispatcherctl is not supported with sqlite3; use a PostgreSQL database')
else:
logger.info('Using config generated from awx.main.dispatch.config.get_dispatcherd_config')
dispatcher_setup(get_dispatcherd_config())
schema_namespace = argparse.Namespace(**options)
data = _build_command_data_from_args(schema_namespace, command)
ctl = get_control_from_settings()
returned = ctl.control_with_reply(command, data=data, expected_replies=expected_replies)
self.stdout.write(yaml.dump(returned, default_flow_style=False))
if len(returned) < expected_replies:
logger.error(f'Obtained only {len(returned)} of {expected_replies}, exiting with non-zero code')
raise CommandError('dispatcherctl returned fewer replies than expected')

View File

@@ -1,85 +0,0 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved
import copy
import hashlib
import json
import logging
import logging.config
import os
from django.conf import settings
from django.core.cache import cache as django_cache
from django.core.management.base import BaseCommand, CommandError
from django.db import connection
from dispatcherd.config import setup as dispatcher_setup
from awx.main.dispatch.config import get_dispatcherd_config
logger = logging.getLogger('awx.main.dispatch')
from dispatcherd import run_service
def _json_default(value):
if isinstance(value, set):
return sorted(value)
if isinstance(value, tuple):
return list(value)
return str(value)
def _hash_config(config):
serialized = json.dumps(config, sort_keys=True, separators=(',', ':'), default=_json_default)
return hashlib.sha256(serialized.encode('utf-8')).hexdigest()
def ensure_no_dispatcherd_env_config():
if os.getenv('DISPATCHERD_CONFIG_FILE'):
raise CommandError('DISPATCHERD_CONFIG_FILE is set but awx-manage dispatcherd uses dynamic config from code')
class Command(BaseCommand):
help = (
'Run the background task service, this is the supported entrypoint since the introduction of dispatcherd as a library. '
'This replaces the prior awx-manage run_dispatcher service, and control actions are at awx-manage dispatcherctl.'
)
def add_arguments(self, parser):
return
def handle(self, *arg, **options):
ensure_no_dispatcherd_env_config()
self.configure_dispatcher_logging()
config = get_dispatcherd_config(for_service=True)
config_hash = _hash_config(config)
logger.info(
'Using dispatcherd config generated from awx.main.dispatch.config.get_dispatcherd_config (sha256=%s)',
config_hash,
)
# Close the connection, because the pg_notify broker will create new async connection
connection.close()
django_cache.close()
dispatcher_setup(config)
run_service()
def configure_dispatcher_logging(self):
# Apply special log rule for the parent process
special_logging = copy.deepcopy(settings.LOGGING)
changed_handlers = []
for handler_name, handler_config in special_logging.get('handlers', {}).items():
filters = handler_config.get('filters', [])
if 'dynamic_level_filter' in filters:
handler_config['filters'] = [flt for flt in filters if flt != 'dynamic_level_filter']
changed_handlers.append(handler_name)
logger.info(f'Dispatcherd main process replaced log level filter for handlers: {changed_handlers}')
# Apply the custom logging level here, before the asyncio code starts
special_logging.setdefault('loggers', {}).setdefault('dispatcherd', {})
special_logging['loggers']['dispatcherd']['level'] = settings.LOG_AGGREGATOR_LEVEL
logging.config.dictConfig(special_logging)

View File

@@ -409,12 +409,10 @@ class Command(BaseCommand):
del_child_group_pks = list(set(db_children_name_pk_map.values()))
for offset in range(0, len(del_child_group_pks), self._batch_size):
child_group_pks = del_child_group_pks[offset : (offset + self._batch_size)]
children_to_remove = list(db_children.filter(pk__in=child_group_pks))
if children_to_remove:
group_group_count += len(children_to_remove)
db_group.children.remove(*children_to_remove)
for db_child in children_to_remove:
logger.debug('Group "%s" removed from group "%s"', db_child.name, db_group.name)
for db_child in db_children.filter(pk__in=child_group_pks):
group_group_count += 1
db_group.children.remove(db_child)
logger.debug('Group "%s" removed from group "%s"', db_child.name, db_group.name)
# FIXME: Inventory source group relationships
# Delete group/host relationships not present in imported data.
db_hosts = db_group.hosts
@@ -443,12 +441,12 @@ class Command(BaseCommand):
del_host_pks = list(del_host_pks)
for offset in range(0, len(del_host_pks), self._batch_size):
del_pks = del_host_pks[offset : (offset + self._batch_size)]
hosts_to_remove = list(db_hosts.filter(pk__in=del_pks))
if hosts_to_remove:
group_host_count += len(hosts_to_remove)
db_group.hosts.remove(*hosts_to_remove)
for db_host in hosts_to_remove:
logger.debug('Host "%s" removed from group "%s"', db_host.name, db_group.name)
for db_host in db_hosts.filter(pk__in=del_pks):
group_host_count += 1
if db_host not in db_group.hosts.all():
continue
db_group.hosts.remove(db_host)
logger.debug('Host "%s" removed from group "%s"', db_host.name, db_group.name)
if settings.SQL_DEBUG:
logger.warning(
'group-group and group-host deletions took %d queries for %d relationships',

View File

@@ -3,6 +3,7 @@
import redis
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
import redis.exceptions
@@ -35,7 +36,11 @@ class Command(BaseCommand):
raise CommandError(f'Callback receiver could not connect to redis, error: {exc}')
try:
consumer = AWXConsumerRedis('callback_receiver', CallbackBrokerWorker())
consumer = AWXConsumerRedis(
'callback_receiver',
CallbackBrokerWorker(),
queues=[getattr(settings, 'CALLBACK_QUEUE', '')],
)
consumer.run()
except KeyboardInterrupt:
print('Terminating Callback Receiver')

View File

@@ -1,20 +1,26 @@
# Copyright (c) 2015 Ansible, Inc.
# All Rights Reserved.
import logging
import logging.config
import yaml
import copy
from django.core.management.base import CommandError
from django.conf import settings
from django.core.management.base import BaseCommand, CommandError
from django.core.cache import cache as django_cache
from django.db import connection
from dispatcherd.factories import get_control_from_settings
from dispatcherd import run_service
from dispatcherd.config import setup as dispatcher_setup
from awx.main.management.commands.dispatcherd import Command as DispatcherdCommand
from awx.main.dispatch.config import get_dispatcherd_config
logger = logging.getLogger('awx.main.dispatch')
class Command(DispatcherdCommand):
help = 'Launch the task dispatcher (deprecated; use awx-manage dispatcherd)'
class Command(BaseCommand):
help = 'Launch the task dispatcher'
def add_arguments(self, parser):
parser.add_argument('--status', dest='status', action='store_true', help='print the internal state of any running dispatchers')
@@ -28,10 +34,8 @@ class Command(DispatcherdCommand):
'Only running tasks can be canceled, queued tasks must be started before they can be canceled.'
),
)
super().add_arguments(parser)
def handle(self, *args, **options):
logger.warning('awx-manage run_dispatcher is deprecated; use awx-manage dispatcherd')
def handle(self, *arg, **options):
if options.get('status'):
ctl = get_control_from_settings()
running_data = ctl.control_with_reply('status')
@@ -61,4 +65,28 @@ class Command(DispatcherdCommand):
results.append(result)
print(yaml.dump(results, default_flow_style=False))
return
return super().handle(*args, **options)
self.configure_dispatcher_logging()
# Close the connection, because the pg_notify broker will create new async connection
connection.close()
django_cache.close()
dispatcher_setup(get_dispatcherd_config(for_service=True))
run_service()
dispatcher_setup(get_dispatcherd_config(for_service=True))
run_service()
def configure_dispatcher_logging(self):
# Apply special log rule for the parent process
special_logging = copy.deepcopy(settings.LOGGING)
for handler_name, handler_config in special_logging.get('handlers', {}).items():
filters = handler_config.get('filters', [])
if 'dynamic_level_filter' in filters:
handler_config['filters'] = [flt for flt in filters if flt != 'dynamic_level_filter']
logger.info(f'Dispatcherd main process replaced log level filter for {handler_name} handler')
# Apply the custom logging level here, before the asyncio code starts
special_logging.setdefault('loggers', {}).setdefault('dispatcherd', {})
special_logging['loggers']['dispatcherd']['level'] = settings.LOG_AGGREGATOR_LEVEL
logging.config.dictConfig(special_logging)

View File

@@ -5,7 +5,6 @@ import logging
import uuid
from django.db import models
from django.conf import settings
from django.db.models import OuterRef, Subquery
from django.db.models.functions import Lower
from ansible_base.lib.utils.db import advisory_lock
@@ -24,65 +23,7 @@ class DeferJobCreatedManager(models.Manager):
return super(DeferJobCreatedManager, self).get_queryset().defer('job_created')
class HostLatestSummaryQuerySet(models.QuerySet):
"""Queryset that annotates and bulk-attaches the latest JobHostSummary
at queryset evaluation time, similar to prefetch_related().
Why not use Django's Prefetch?
Django's Prefetch with [:1] slicing fetches 1 record globally, not per-host
(Django ticket #26780). Window-function workarounds require Django 4.2+ and
are more complex. Prefetching all summaries then filtering in Python wastes
memory for hosts with many job runs. The approach here — annotate the latest
ID via Subquery, then in_bulk() only those IDs — is the same 2-query pattern
prefetch_related uses internally, customized for "latest per group."
Not streaming-safe: relies on _result_cache existing after _fetch_all().
"""
_awx_latest_summary_attached = False
def _clone(self):
clone = super()._clone()
clone._awx_latest_summary_attached = self._awx_latest_summary_attached
return clone
def with_latest_summary_id(self):
from awx.main.models.jobs import JobHostSummary
latest_summary = JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id')
return self.annotate(
_latest_summary_id=Subquery(latest_summary.values('id')[:1]),
)
def _fetch_all(self):
super()._fetch_all()
if self._awx_latest_summary_attached or not self._result_cache:
return
# Only bulk-attach if the queryset was annotated via with_latest_summary_id().
# Without this guard, we'd set _latest_summary_cache=None on every host,
# masking the per-object fallback query in Host.latest_summary.
if not hasattr(self._result_cache[0], '_latest_summary_id'):
return
from awx.main.models.jobs import JobHostSummary
latest_summary_ids = [host._latest_summary_id for host in self._result_cache if host._latest_summary_id is not None]
if latest_summary_ids:
summaries_by_id = JobHostSummary.objects.select_related('job', 'job__job_template').in_bulk(latest_summary_ids)
else:
summaries_by_id = {}
for host in self._result_cache:
latest_summary_id = getattr(host, '_latest_summary_id', None)
host._latest_summary_cache = summaries_by_id.get(latest_summary_id)
self._awx_latest_summary_attached = True
class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)):
class HostManager(models.Manager):
"""Custom manager class for Hosts model."""
def active_count(self):
@@ -90,46 +31,38 @@ class HostManager(models.Manager.from_queryset(HostLatestSummaryQuerySet)):
Construction of query involves:
- remove any ordering specified in model's Meta
- Exclude hosts sourced from another Tower
- Exclude hosts in constructed inventories (these are shadow rows of source-inventory hosts)
- Restrict the query to only return the name column
- Only consider results that are unique
- Return the count of this query
"""
return (
self.order_by()
.exclude(inventory_sources__source='controller')
.exclude(inventory__kind='constructed')
.values(name_lower=Lower('name'))
.distinct()
.count()
)
return self.order_by().exclude(inventory_sources__source='controller').values(name_lower=Lower('name')).distinct().count()
def org_active_count(self, org_id):
"""Return count of active, unique hosts used by an organization.
Construction of query involves:
- remove any ordering specified in model's Meta
- Exclude hosts sourced from another Tower
- Exclude hosts in constructed inventories (these are shadow rows of source-inventory hosts)
- Consider only hosts where the canonical inventory is owned by the organization
- Restrict the query to only return the name column
- Only consider results that are unique
- Return the count of this query
"""
return (
self.order_by()
.exclude(inventory_sources__source='controller')
.exclude(inventory__kind='constructed')
.filter(inventory__organization=org_id)
.values('name')
.distinct()
.count()
)
return self.order_by().exclude(inventory_sources__source='controller').filter(inventory__organization=org_id).values('name').distinct().count()
def get_queryset(self):
"""When the parent instance of the host query set has a `kind=smart` and a `host_filter`
set. Use the `host_filter` to generate the queryset for the hosts.
"""
qs = super().get_queryset().defer('ansible_facts')
qs = (
super(HostManager, self)
.get_queryset()
.defer(
'last_job__extra_vars',
'last_job_host_summary__job__extra_vars',
'last_job__artifacts',
'last_job_host_summary__job__artifacts',
)
)
if hasattr(self, 'instance') and hasattr(self.instance, 'host_filter') and hasattr(self.instance, 'kind'):
if self.instance.kind == 'smart' and self.instance.host_filter is not None:

View File

@@ -21,6 +21,6 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(setup_tower_managed_defaults, migrations.RunPython.noop),
migrations.RunPython(setup_rbac_role_system_administrator, migrations.RunPython.noop),
migrations.RunPython(setup_tower_managed_defaults),
migrations.RunPython(setup_rbac_role_system_administrator),
]

View File

@@ -98,5 +98,5 @@ class Migration(migrations.Migration):
]
operations = [
migrations.RunPython(convert_controller_role_definitions, migrations.RunPython.noop),
migrations.RunPython(convert_controller_role_definitions),
]

View File

@@ -3,15 +3,19 @@ from django.db import migrations, models
from awx.main.migrations._create_system_jobs import delete_clear_tokens_sjt
# --- START of function merged from 0203_rename_github_app_kind.py ---
def update_github_app_kind(apps, schema_editor):
"""
Updates the 'namespace' field for CredentialType records
Updates the 'kind' field for CredentialType records
from 'github_app' to 'github_app_lookup'.
This addresses a change in the entry point key for the GitHub App plugin.
"""
CredentialType = apps.get_model('main', 'CredentialType')
db_alias = schema_editor.connection.alias
CredentialType.objects.using(db_alias).filter(namespace='github_app').update(namespace='github_app_lookup')
CredentialType.objects.using(db_alias).filter(kind='github_app').update(kind='github_app_lookup')
# --- END of function merged from 0203_rename_github_app_kind.py ---
class Migration(migrations.Migration):
@@ -114,5 +118,7 @@ class Migration(migrations.Migration):
max_length=32,
),
),
# --- START of operations merged from 0203_rename_github_app_kind.py ---
migrations.RunPython(update_github_app_kind, migrations.RunPython.noop),
# --- END of operations merged from 0203_rename_github_app_kind.py ---
]

View File

@@ -1,29 +0,0 @@
# Generated by Django 5.2.8 on 2026-02-20 03:39
from django.db import migrations
class Migration(migrations.Migration):
dependencies = [
('main', '0204_squashed_deletions'),
]
operations = [
migrations.AlterModelOptions(
name='instancegroup',
options={
'default_permissions': ('change', 'delete', 'view'),
'ordering': ('pk',),
'permissions': [('use_instancegroup', 'Can use instance group in a preference list of a resource')],
},
),
migrations.AlterModelOptions(
name='workflowjobnode',
options={'ordering': ('pk',)},
),
migrations.AlterModelOptions(
name='workflowjobtemplatenode',
options={'ordering': ('pk',)},
),
]

View File

@@ -386,6 +386,7 @@ class gce(PluginFileInjector):
# auth related items
ret['auth_kind'] = "serviceaccount"
filters = []
# TODO: implement gce group_by options
# gce never processed the group_by field, if it had, we would selectively
# apply those options here, but it did not, so all groups are added here
@@ -419,6 +420,8 @@ class gce(PluginFileInjector):
if keyed_groups:
ret['keyed_groups'] = keyed_groups
if filters:
ret['filters'] = filters
if compose_dict:
ret['compose'] = compose_dict
if inventory_source.source_regions and 'all' not in inventory_source.source_regions:

View File

@@ -315,11 +315,12 @@ class PrimordialModel(HasEditsMixin, CreatedModifiedModel):
)
def __init__(self, *args, **kwargs):
super(PrimordialModel, self).__init__(*args, **kwargs)
r = super(PrimordialModel, self).__init__(*args, **kwargs)
if self.pk:
self._prior_values_store = self._get_fields_snapshot()
else:
self._prior_values_store = {}
return r
def save(self, *args, **kwargs):
update_fields = kwargs.get('update_fields', [])

View File

@@ -28,7 +28,6 @@ from rest_framework.serializers import ValidationError as DRFValidationError
from ansible_base.lib.utils.db import advisory_lock
# AWX
from awx.main.constants import OIDC_CREDENTIAL_TYPE_NAMESPACES
from awx.api.versioning import reverse
from awx.main.fields import (
ImplicitRoleField,
@@ -49,6 +48,10 @@ from awx.main.models import Team, Organization
from awx.main.utils import encrypt_field
from awx_plugins.interfaces._temporary_private_licensing_api import detect_server_product_name
# DAB
from ansible_base.resource_registry.tasks.sync import get_resource_server_client
from ansible_base.resource_registry.utils.settings import resource_server_defined
__all__ = ['Credential', 'CredentialType', 'CredentialInputSource', 'build_safe_env']
logger = logging.getLogger('awx.main.models.credential')
@@ -76,6 +79,46 @@ def build_safe_env(env):
return safe_env
def check_resource_server_for_user_in_organization(user, organization, requesting_user):
if not resource_server_defined():
return False
if not requesting_user:
return False
client = get_resource_server_client(settings.RESOURCE_SERVICE_PATH, jwt_user_id=str(requesting_user.resource.ansible_id), raise_if_bad_request=False)
# need to get the organization object_id in resource server, by querying with ansible_id
response = client._make_request(path=f'resources/?ansible_id={str(organization.resource.ansible_id)}', method='GET')
response_json = response.json()
if response.status_code != 200:
logger.error(f'Failed to get organization object_id in resource server: {response_json.get("detail", "")}')
return False
if response_json.get('count', 0) == 0:
return False
org_id_in_resource_server = response_json['results'][0]['object_id']
client.base_url = client.base_url.replace('/api/gateway/v1/service-index/', '/api/gateway/v1/')
# find role assignments with:
# - roles Organization Member or Organization Admin
# - user ansible id
# - organization object id
response = client._make_request(
path=f'role_user_assignments/?role_definition__name__in=Organization Member,Organization Admin&user__resource__ansible_id={str(user.resource.ansible_id)}&object_id={org_id_in_resource_server}',
method='GET',
)
response_json = response.json()
if response.status_code != 200:
logger.error(f'Failed to get role user assignments in resource server: {response_json.get("detail", "")}')
return False
if response_json.get('count', 0) > 0:
return True
return False
class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
"""
A credential contains information about how to talk to a remote resource
@@ -199,29 +242,6 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
needed.append('vault_password')
return needed
@functools.cached_property
def context(self):
"""
Property for storing runtime context during credential resolution.
The context is a dict keyed by CredentialInputSource PK, where each value
is a dict of runtime fields for that input source. Example::
{
<input_source_pk>: {
"workload_identity_token": "<jwt_token>"
},
<another_input_source_pk>: {
"workload_identity_token": "<different_jwt_token>"
},
}
This structure allows each input source to have its own set of runtime
values, avoiding conflicts when a credential has multiple input sources
with different configurations (e.g., different JWT audiences).
"""
return {}
@cached_property
def dynamic_input_fields(self):
# if the credential is not yet saved we can't access the input_sources
@@ -347,20 +367,21 @@ class Credential(PasswordFieldsModel, CommonModelNameNotUnique, ResourceMixin):
def _get_dynamic_input(self, field_name):
for input_source in self.input_sources.all():
if input_source.input_field_name == field_name:
return input_source.get_input_value(context=self.context)
return input_source.get_input_value()
else:
raise ValueError('{} is not a dynamic input field'.format(field_name))
def validate_role_assignment(self, actor, role_definition, **kwargs):
requesting_user = kwargs.get('requesting_user', None)
if requesting_user and requesting_user.is_superuser:
return
if self.organization:
if isinstance(actor, User):
if actor.is_superuser:
return
if Organization.access_qs(actor, 'member').filter(id=self.organization.id).exists():
return
requesting_user = kwargs.get('requesting_user', None)
if check_resource_server_for_user_in_organization(actor, self.organization, requesting_user):
return
if isinstance(actor, Team):
if actor.organization == self.organization:
return
@@ -414,15 +435,13 @@ class CredentialType(CommonModelNameNotUnique):
def from_db(cls, db, field_names, values):
instance = super(CredentialType, cls).from_db(db, field_names, values)
if instance.managed and instance.namespace and instance.kind != "external":
native = ManagedCredentialType.registry.get(instance.namespace)
if native:
instance.inputs = native.inputs
instance.injectors = native.injectors
instance.custom_injectors = getattr(native, 'custom_injectors', None)
native = ManagedCredentialType.registry[instance.namespace]
instance.inputs = native.inputs
instance.injectors = native.injectors
instance.custom_injectors = getattr(native, 'custom_injectors', None)
elif instance.namespace and instance.kind == "external":
native = ManagedCredentialType.registry.get(instance.namespace)
if native:
instance.inputs = native.inputs
native = ManagedCredentialType.registry[instance.namespace]
instance.inputs = native.inputs
return instance
@@ -486,7 +505,6 @@ class CredentialType(CommonModelNameNotUnique):
existing = ct_class.objects.filter(name=default.name, kind=default.kind).first()
if existing is not None:
existing.namespace = default.namespace
existing.description = getattr(default, 'description', '')
existing.inputs = {}
existing.injectors = {}
existing.save()
@@ -526,14 +544,7 @@ class CredentialType(CommonModelNameNotUnique):
@classmethod
def load_plugin(cls, ns, plugin):
# TODO: User "side-loaded" credential custom_injectors isn't supported
ManagedCredentialType.registry[ns] = SimpleNamespace(
namespace=ns,
name=plugin.name,
kind='external',
inputs=plugin.inputs,
backend=plugin.backend,
description=getattr(plugin, 'plugin_description', ''),
)
ManagedCredentialType.registry[ns] = SimpleNamespace(namespace=ns, name=plugin.name, kind='external', inputs=plugin.inputs, backend=plugin.backend)
def inject_credential(self, credential, env, safe_env, args, private_data_dir, container_root=None):
from awx_plugins.interfaces._temporary_private_inject_api import inject_credential
@@ -545,13 +556,7 @@ class CredentialTypeHelper:
@classmethod
def get_creation_params(cls, cred_type):
if cred_type.kind == 'external':
return {
'namespace': cred_type.namespace,
'kind': cred_type.kind,
'name': cred_type.name,
'managed': True,
'description': getattr(cred_type, 'description', ''),
}
return dict(namespace=cred_type.namespace, kind=cred_type.kind, name=cred_type.name, managed=True)
return dict(
namespace=cred_type.namespace,
kind=cred_type.kind,
@@ -617,15 +622,7 @@ class CredentialInputSource(PrimordialModel):
raise ValidationError(_('Input field must be defined on target credential (options are {}).'.format(', '.join(sorted(defined_fields)))))
return self.input_field_name
def get_input_value(self, context: dict | None = None):
"""
Retrieve the value from the external credential backend.
Args:
context: Optional runtime context dict passed from the target credential.
"""
if context is None:
context = {}
def get_input_value(self):
backend = self.source_credential.credential_type.plugin.backend
backend_kwargs = {}
for field_name, value in self.source_credential.inputs.items():
@@ -636,17 +633,6 @@ class CredentialInputSource(PrimordialModel):
backend_kwargs.update(self.metadata)
# Resolve internal fields from the per-input-source context.
# The context dict is keyed by input source PK, e.g.:
# {42: {"workload_identity_token": "eyJ..."}, 43: {"workload_identity_token": "eyX..."}}
# This allows each input source to carry its own runtime values.
input_source_context = context.get(self.pk, {})
for field in self.source_credential.credential_type.inputs.get('fields', []):
if field.get('internal'):
value = input_source_context.get(field['id'])
if value is not None:
backend_kwargs[field['id']] = value
with set_environ(**settings.AWX_TASK_ENV):
return backend(**backend_kwargs)
@@ -655,20 +641,13 @@ class CredentialInputSource(PrimordialModel):
return reverse(view_name, kwargs={'pk': self.pk}, request=request)
def _is_oidc_namespace_disabled(ns):
"""Check if a credential namespace should be skipped based on the OIDC feature flag."""
return ns in OIDC_CREDENTIAL_TYPE_NAMESPACES and not getattr(settings, 'FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED', False)
def load_credentials():
awx_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials')}
supported_entry_points = {ep.name: ep for ep in entry_points(group='awx_plugins.managed_credentials.supported')}
plugin_entry_points = awx_entry_points if detect_server_product_name() == 'AWX' else {**awx_entry_points, **supported_entry_points}
for ns, ep in plugin_entry_points.items():
if _is_oidc_namespace_disabled(ns):
continue
cred_plugin = ep.load()
if not hasattr(cred_plugin, 'inputs'):
setattr(cred_plugin, 'inputs', {})
@@ -687,8 +666,5 @@ def load_credentials():
credential_plugins = {}
for ns, ep in credential_plugins.items():
if _is_oidc_namespace_disabled(ns):
continue
plugin = ep.load()
CredentialType.load_plugin(ns, plugin)

View File

@@ -24,6 +24,7 @@ from awx.main.managers import DeferJobCreatedManager
from awx.main.constants import MINIMAL_EVENTS
from awx.main.models.base import CreatedModifiedModel
from awx.main.utils import ignore_inventory_computed_fields, camelcase_to_underscore
from awx.main.utils.db import bulk_update_sorted_by_id
analytics_logger = logging.getLogger('awx.analytics.job_events')
@@ -589,8 +590,20 @@ class JobEvent(BasePlaybookEvent):
JobHostSummary.objects.bulk_create(summaries.values())
# last_job and last_job_host_summary are now derived via
# JobHostSummary.latest_for_host / latest_job_for_host
# update the last_job_id and last_job_host_summary_id
# in single queries
host_mapping = dict((summary['host_id'], summary['id']) for summary in JobHostSummary.objects.filter(job_id=job.id).values('id', 'host_id'))
updated_hosts = set()
for h in all_hosts:
# if the hostname *shows up* in the playbook_on_stats event
if h.name in hostnames:
h.last_job_id = job.id
updated_hosts.add(h)
if h.id in host_mapping:
h.last_job_host_summary_id = host_mapping[h.id]
updated_hosts.add(h)
bulk_update_sorted_by_id(Host, updated_hosts, ['last_job_id', 'last_job_host_summary_id'])
# Create/update Host Metrics
self._update_host_metrics(updated_hosts_list)

View File

@@ -58,6 +58,8 @@ class ExecutionEnvironment(CommonModel):
return reverse('api:execution_environment_detail', kwargs={'pk': self.pk}, request=request)
def validate_role_assignment(self, actor, role_definition, **kwargs):
from awx.main.models.credential import check_resource_server_for_user_in_organization
if self.managed:
raise ValidationError({'object_id': _('Can not assign object roles to managed Execution Environments')})
if self.organization_id is None:
@@ -67,4 +69,8 @@ class ExecutionEnvironment(CommonModel):
if actor.has_obj_perm(self.organization, 'view'):
return
requesting_user = kwargs.get('requesting_user', None)
if check_resource_server_for_user_in_organization(actor, self.organization, requesting_user):
return
raise ValidationError({'user': _('User must have view permission to Execution Environment organization')})

View File

@@ -50,8 +50,9 @@ class HasPolicyEditsMixin(HasEditsMixin):
abstract = True
def __init__(self, *args, **kwargs):
super(BaseModel, self).__init__(*args, **kwargs)
r = super(BaseModel, self).__init__(*args, **kwargs)
self._prior_values_store = self._get_fields_snapshot()
return r
def save(self, *args, **kwargs):
super(BaseModel, self).save(*args, **kwargs)
@@ -485,7 +486,6 @@ class InstanceGroup(HasPolicyEditsMixin, BaseModel, RelatedJobsMixin, ResourceMi
class Meta:
app_label = 'main'
ordering = ('pk',)
permissions = [('use_instancegroup', 'Can use instance group in a preference list of a resource')]
# Since this has no direct organization field only superuser can add, so remove add permission
default_permissions = ('change', 'delete', 'view')

View File

@@ -18,7 +18,7 @@ from django.db import transaction
from django.core.exceptions import ValidationError
from django.urls import resolve
from django.utils.timezone import now
from django.db.models import Q, Subquery, OuterRef
from django.db.models import Q
# REST Framework
from rest_framework.exceptions import ParseError
@@ -386,10 +386,7 @@ class Inventory(CommonModelNameNotUnique, ResourceMixin, RelatedJobsMixin, OpaQu
logger.debug("Going to update inventory computed fields, pk={0}".format(self.pk))
start_time = time.time()
active_hosts = self.hosts
from awx.main.models.jobs import JobHostSummary # circular import: inventory.py loads before jobs.py
latest_summary_failed = Subquery(JobHostSummary.objects.filter(host_id=OuterRef('pk')).order_by('-id').values('failed')[:1])
failed_hosts = active_hosts.annotate(_latest_failed=latest_summary_failed).filter(_latest_failed=True)
failed_hosts = active_hosts.filter(last_job_host_summary__failed=True)
active_groups = self.groups
if self.kind == 'smart':
active_groups = active_groups.none()
@@ -585,23 +582,6 @@ class Host(CommonModelNameNotUnique, RelatedJobsMixin):
objects = HostManager()
@property
def latest_summary(self):
if hasattr(self, '_latest_summary_cache'):
return self._latest_summary_cache
from awx.main.models.jobs import JobHostSummary
summary = JobHostSummary.objects.filter(host_id=self.pk).order_by('-id').select_related('job', 'job__job_template').first()
self._latest_summary_cache = summary
return summary
@property
def latest_job(self):
summary = self.latest_summary
if summary is None:
return None
return summary.job
def get_absolute_url(self, request=None):
return reverse('api:host_detail', kwargs={'pk': self.pk}, request=request)

View File

@@ -52,7 +52,7 @@ from awx.main.models.mixins import (
WebhookTemplateMixin,
OpaQueryPathMixin,
)
from awx.main.utils.common import get_job_variable_prefixes
from awx.main.constants import JOB_VARIABLE_PREFIXES
logger = logging.getLogger('awx.main.models.jobs')
@@ -817,20 +817,19 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
def awx_meta_vars(self):
r = super(Job, self).awx_meta_vars()
prefixes = get_job_variable_prefixes()
if self.project:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_project_revision'.format(name)] = self.project.scm_revision
r['{}_project_scm_branch'.format(name)] = self.project.scm_branch
if self.scm_branch:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_job_scm_branch'.format(name)] = self.scm_branch
if self.job_template:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_job_template_id'.format(name)] = self.job_template.pk
r['{}_job_template_name'.format(name)] = self.job_template.name
if self.execution_node:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_execution_node'.format(name)] = self.execution_node
return r
@@ -846,21 +845,6 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
def get_notification_friendly_name(self):
return "Job"
def get_source_hosts_for_constructed_inventory(self):
"""Return a QuerySet of the source (input inventory) hosts for a constructed inventory.
Constructed inventory hosts have an instance_id pointing to the real
host in the input inventory. This resolves those references and returns
a proper QuerySet (never a list), suitable for use with finish_fact_cache.
"""
Host = JobHostSummary._meta.get_field('host').related_model
if not self.inventory_id:
return Host.objects.none()
id_field = Host._meta.get_field('id')
return Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field))).only(
*HOST_FACTS_FIELDS
)
def get_hosts_for_fact_cache(self):
"""
Builds the queryset to use for writing or finalizing the fact cache
@@ -868,15 +852,17 @@ class Job(UnifiedJob, JobOptions, SurveyJobMixin, JobNotificationMixin, TaskMana
For constructed inventories, that means the original (input inventory) hosts
when slicing, that means only returning hosts in that slice
"""
Host = JobHostSummary._meta.get_field('host').related_model
if not self.inventory_id:
Host = JobHostSummary._meta.get_field('host').related_model
return Host.objects.none()
if self.inventory.kind == 'constructed':
host_qs = self.get_source_hosts_for_constructed_inventory()
id_field = Host._meta.get_field('id')
host_qs = Host.objects.filter(id__in=self.inventory.hosts.exclude(instance_id='').values_list(Cast('instance_id', output_field=id_field)))
else:
host_qs = self.inventory.hosts.only(*HOST_FACTS_FIELDS)
host_qs = self.inventory.hosts
host_qs = host_qs.only(*HOST_FACTS_FIELDS)
host_qs = self.inventory.get_sliced_hosts(host_qs, self.job_slice_number, self.job_slice_count)
return host_qs
@@ -1141,22 +1127,6 @@ class JobHostSummary(CreatedModifiedModel):
self.skipped,
)
@classmethod
def latest_for_host(cls, host_id):
"""Return the most recent JobHostSummary for a given host, or None."""
return cls.objects.filter(host_id=host_id).order_by('-id').first()
@classmethod
def latest_job_for_host(cls, host_id):
"""Return the Job from the most recent JobHostSummary for a host, or None."""
summary = cls.latest_for_host(host_id)
if summary:
try:
return summary.job
except cls.job.field.related_model.DoesNotExist:
return None
return None
def get_absolute_url(self, request=None):
return reverse('api:job_host_summary_detail', kwargs={'pk': self.pk}, request=request)

View File

@@ -188,16 +188,6 @@ class SurveyJobTemplateMixin(models.Model):
runtime_extra_vars.pop(variable_key)
if default is not None:
# do not add variables that contain an empty string, are not required and are not present in extra_vars
# password fields must be skipped, because default values have special behaviour
if (
default == ''
and not survey_element.get('required')
and survey_element.get('type') != 'password'
and variable_key not in runtime_extra_vars
):
continue
decrypted_default = default
if survey_element['type'] == "password" and isinstance(decrypted_default, str) and decrypted_default.startswith('$encrypted$'):
decrypted_default = decrypt_value(get_encryption_key('value', pk=None), decrypted_default)

View File

@@ -72,10 +72,10 @@ def _fast_forward_rrule(rrule, ref_dt=None):
if ref_dt is None:
ref_dt = now()
dtstart_tz = rrule._dtstart.tzinfo
ref_dt = ref_dt.astimezone(dtstart_tz)
ref_dt = ref_dt.astimezone(datetime.timezone.utc)
if rrule._dtstart > ref_dt:
rrule_dtstart_utc = rrule._dtstart.astimezone(datetime.timezone.utc)
if rrule_dtstart_utc > ref_dt:
return rrule
interval = rrule._interval if rrule._interval else 1
@@ -84,14 +84,20 @@ def _fast_forward_rrule(rrule, ref_dt=None):
elif rrule._freq == dateutil.rrule.MINUTELY:
interval *= 60
# if after converting to seconds the interval is still a fraction,
# just return original rrule
if isinstance(interval, float) and not interval.is_integer():
return rrule
seconds_since_dtstart = (ref_dt - rrule._dtstart).total_seconds()
seconds_since_dtstart = (ref_dt - rrule_dtstart_utc).total_seconds()
# it is important to fast forward by a number that is divisible by
# interval. For example, if interval is 7 hours, we fast forward by 7, 14, 21, etc. hours.
# Otherwise, the occurrences after the fast forward might not match the ones before.
# x // y is integer division, lopping off any remainder, so that we get the outcome we want.
interval_aligned_offset = datetime.timedelta(seconds=(seconds_since_dtstart // interval) * interval)
new_start = rrule._dtstart + interval_aligned_offset
new_rrule = rrule.replace(dtstart=new_start)
new_start = rrule_dtstart_utc + interval_aligned_offset
new_rrule = rrule.replace(dtstart=new_start.astimezone(rrule._dtstart.tzinfo))
return new_rrule

View File

@@ -10,6 +10,7 @@ import json
import logging
import os
import re
import socket
import subprocess
import tempfile
from collections import OrderedDict
@@ -58,8 +59,7 @@ from awx.main.utils.common import (
)
from awx.main.utils.encryption import encrypt_dict, decrypt_field
from awx.main.utils import polymorphic
from awx.main.constants import ACTIVE_STATES, CAN_CANCEL
from awx.main.utils.common import get_job_variable_prefixes
from awx.main.constants import ACTIVE_STATES, CAN_CANCEL, JOB_VARIABLE_PREFIXES
from awx.main.redact import UriCleaner, REPLACE_STR
from awx.main.consumers import emit_channel_notification
from awx.main.fields import AskForField, OrderedManyToManyField
@@ -919,7 +919,7 @@ class UnifiedJob(
# If we have a start and finished time, and haven't already calculated
# out the time that elapsed, do so.
if self.started and self.finished and self.elapsed == decimal.Decimal(0):
if self.started and self.finished and self.elapsed == 0.0:
td = self.finished - self.started
elapsed = decimal.Decimal(td.total_seconds())
self.elapsed = elapsed.quantize(dq)
@@ -1355,6 +1355,8 @@ class UnifiedJob(
status_data['instance_group_name'] = None
elif status in ['successful', 'failed', 'canceled'] and self.finished:
status_data['finished'] = datetime.datetime.strftime(self.finished, "%Y-%m-%dT%H:%M:%S.%fZ")
elif status == 'running':
status_data['started'] = datetime.datetime.strftime(self.finished, "%Y-%m-%dT%H:%M:%S.%fZ")
status_data.update(self.websocket_emit_data())
status_data['group_name'] = 'jobs'
if getattr(self, 'unified_job_template_id', None):
@@ -1486,17 +1488,40 @@ class UnifiedJob(
return 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (self.model_to_str(), self.name, self.id)
return None
def fallback_cancel(self):
if not self.celery_task_id:
self.refresh_from_db(fields=['celery_task_id'])
self.cancel_dispatcher_process()
def cancel_dispatcher_process(self):
"""Returns True if dispatcher running this job acknowledged request and sent SIGTERM"""
if not self.celery_task_id:
return False
# Special case for task manager (used during workflow job cancellation)
if not connection.get_autocommit():
try:
ctl = get_control_from_settings()
ctl.control('cancel', data={'uuid': self.celery_task_id})
except Exception:
logger.exception("Error sending cancel command to dispatcher")
return True # task manager itself needs to act under assumption that cancel was received
# Standard case with reply
try:
logger.info(f'Sending cancel message to pg_notify channel {self.controller_node} for task {self.celery_task_id}')
ctl = get_control_from_settings(default_publish_channel=self.controller_node)
ctl.control('cancel', data={'uuid': self.celery_task_id})
timeout = 5
ctl = get_control_from_settings()
results = ctl.control_with_reply('cancel', data={'uuid': self.celery_task_id}, expected_replies=1, timeout=timeout)
# Check if cancel was successful by checking if we got any results
return bool(results and len(results) > 0)
except socket.timeout:
logger.error(f'could not reach dispatcher on {self.controller_node} within {timeout}s')
except Exception:
logger.exception("Error sending cancel command to dispatcher")
logger.exception("error encountered when checking task status")
return False # whether confirmation was obtained
def cancel(self, job_explanation=None, is_chain=False):
if self.can_cancel:
@@ -1519,13 +1544,19 @@ class UnifiedJob(
# the job control process will use the cancel_flag to distinguish a shutdown from a cancel
self.save(update_fields=cancel_fields)
# Be extra sure we have the task id, in case job is transitioning into running right now
if not self.celery_task_id:
self.refresh_from_db(fields=['celery_task_id', 'controller_node'])
# send pg_notify message to cancel, will not send until transaction completes
controller_notified = False
if self.celery_task_id:
self.cancel_dispatcher_process()
controller_notified = self.cancel_dispatcher_process()
# If a SIGTERM signal was sent to the control process, and acked by the dispatcher
# then we want to let its own cleanup change status, otherwise change status now
if not controller_notified:
if self.status != 'canceled':
self.status = 'canceled'
self.save(update_fields=['status'])
# Avoid race condition where we have stale model from pending state but job has already started,
# its checking signal but not cancel_flag, so re-send signal after updating cancel fields
self.fallback_cancel()
return self.cancel_flag
@@ -1569,8 +1600,7 @@ class UnifiedJob(
by AWX, for purposes of client playbook hooks
"""
r = {}
prefixes = get_job_variable_prefixes()
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_job_id'.format(name)] = self.pk
r['{}_job_launch_type'.format(name)] = self.launch_type
@@ -1579,7 +1609,7 @@ class UnifiedJob(
wj = self.get_workflow_job()
if wj:
schedule = getattr_dne(wj, 'schedule')
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_workflow_job_id'.format(name)] = wj.pk
r['{}_workflow_job_name'.format(name)] = wj.name
r['{}_workflow_job_launch_type'.format(name)] = wj.launch_type
@@ -1590,12 +1620,12 @@ class UnifiedJob(
if not created_by:
schedule = getattr_dne(self, 'schedule')
if schedule:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_schedule_id'.format(name)] = schedule.pk
r['{}_schedule_name'.format(name)] = schedule.name
if created_by:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_user_id'.format(name)] = created_by.pk
r['{}_user_name'.format(name)] = created_by.username
r['{}_user_email'.format(name)] = created_by.email
@@ -1604,7 +1634,7 @@ class UnifiedJob(
inventory = getattr_dne(self, 'inventory')
if inventory:
for name in prefixes:
for name in JOB_VARIABLE_PREFIXES:
r['{}_inventory_id'.format(name)] = inventory.pk
r['{}_inventory_name'.format(name)] = inventory.name

View File

@@ -200,7 +200,6 @@ class WorkflowJobTemplateNode(WorkflowNodeBase):
indexes = [
models.Index(fields=['identifier']),
]
ordering = ('pk',)
def get_absolute_url(self, request=None):
return reverse('api:workflow_job_template_node_detail', kwargs={'pk': self.pk}, request=request)
@@ -287,7 +286,6 @@ class WorkflowJobNode(WorkflowNodeBase):
models.Index(fields=["identifier", "workflow_job"]),
models.Index(fields=['identifier']),
]
ordering = ('pk',)
@property
def event_processing_finished(self):
@@ -345,11 +343,7 @@ class WorkflowJobNode(WorkflowNodeBase):
)
data.update(accepted_fields) # missing fields are handled in the scheduler
# build ancestor artifacts, save them to node model for later
# initialize from pre-seeded ancestor_artifacts (set on root nodes of
# child workflows via seed_root_ancestor_artifacts to carry artifacts
# from the parent workflow); exclude job_slice which is internal
# metadata handled separately below
aa_dict = {k: v for k, v in self.ancestor_artifacts.items() if k != 'job_slice'} if self.ancestor_artifacts else {}
aa_dict = {}
is_root_node = True
for parent_node in self.get_parent_nodes():
is_root_node = False
@@ -370,13 +364,11 @@ class WorkflowJobNode(WorkflowNodeBase):
data['survey_passwords'] = password_dict
# process extra_vars
extra_vars = data.get('extra_vars', {})
if ujt_obj and isinstance(ujt_obj, JobTemplate):
if ujt_obj and isinstance(ujt_obj, (JobTemplate, WorkflowJobTemplate)):
if aa_dict:
functional_aa_dict = copy(aa_dict)
functional_aa_dict.pop('_ansible_no_log', None)
extra_vars.update(functional_aa_dict)
elif ujt_obj and isinstance(ujt_obj, WorkflowJobTemplate):
pass # artifacts are applied via seed_root_ancestor_artifacts in the task manager
# Workflow Job extra_vars higher precedence than ancestor artifacts
extra_vars.update(wj_special_vars)
@@ -740,18 +732,6 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
wj = wj.get_workflow_job()
return ancestors
def seed_root_ancestor_artifacts(self, artifacts):
"""Apply parent workflow artifacts to root nodes so they propagate
through the normal ancestor_artifacts channel instead of being
baked into this workflow's extra_vars."""
self.workflow_job_nodes.exclude(
workflowjobnodes_success__isnull=False,
).exclude(
workflowjobnodes_failure__isnull=False,
).exclude(
workflowjobnodes_always__isnull=False,
).update(ancestor_artifacts=artifacts)
def get_effective_artifacts(self, **kwargs):
"""
For downstream jobs of a workflow nested inside of a workflow,
@@ -805,7 +785,7 @@ class WorkflowJob(UnifiedJob, WorkflowJobOptions, SurveyJobMixin, JobNotificatio
def cancel_dispatcher_process(self):
# WorkflowJobs don't _actually_ run anything in the dispatcher, so
# there's no point in asking the dispatcher if it knows about this task
return
return True
class WorkflowApprovalTemplate(UnifiedJobTemplate, RelatedJobsMixin):
@@ -936,17 +916,6 @@ class WorkflowApproval(UnifiedJob, JobNotificationMixin):
ScheduleWorkflowManager().schedule()
return reverse('api:workflow_approval_deny', kwargs={'pk': self.pk}, request=request)
def cancel(self, job_explanation=None, is_chain=False):
# WorkflowApprovals have no dispatcher process (they wait for human
# input) and are excluded from TaskManager processing, so the base
# cancel() would only set cancel_flag without ever transitioning the
# status. We call super() for the flag, then transition directly.
has_already_canceled = bool(self.status == 'canceled')
super().cancel(job_explanation=job_explanation, is_chain=is_chain)
if self.status != 'canceled' and not has_already_canceled:
self.status = 'canceled'
self.save(update_fields=['status'])
def signal_start(self, **kwargs):
can_start = super(WorkflowApproval, self).signal_start(**kwargs)
self.started = self.created

View File

@@ -76,12 +76,10 @@ class GrafanaBackend(AWXBaseEmailBackend, CustomNotificationBase):
grafana_headers = {}
if 'started' in m.body:
try:
epoch = datetime.datetime.fromtimestamp(0, tz=datetime.timezone.utc)
grafana_data['time'] = grafana_data['timeEnd'] = int(
(dp.parse(m.body['started']).replace(tzinfo=datetime.timezone.utc) - epoch).total_seconds() * 1000
)
epoch = datetime.datetime.utcfromtimestamp(0)
grafana_data['time'] = grafana_data['timeEnd'] = int((dp.parse(m.body['started']).replace(tzinfo=None) - epoch).total_seconds() * 1000)
if m.body.get('finished'):
grafana_data['timeEnd'] = int((dp.parse(m.body['finished']).replace(tzinfo=datetime.timezone.utc) - epoch).total_seconds() * 1000)
grafana_data['timeEnd'] = int((dp.parse(m.body['finished']).replace(tzinfo=None) - epoch).total_seconds() * 1000)
except ValueError:
logger.error(smart_str(_("Error converting time {} or timeEnd {} to int.").format(m.body['started'], m.body['finished'])))
if not self.fail_silently:

View File

@@ -1,7 +1,6 @@
# Copyright (c) 2016 Ansible, Inc.
# All Rights Reserved.
import base64
import json
import logging
import requests
@@ -85,25 +84,20 @@ class WebhookBackend(AWXBaseEmailBackend, CustomNotificationBase):
if resp.status_code not in [301, 307]:
break
# convert the url to a base64 encoded string for safe logging
url_log_safe = base64.b64encode(url.encode('UTF-8'))
# get the next URL to try
url_next = resp.headers.get("Location", None)
url_next_log_safe = base64.b64encode(url_next.encode('UTF-8')) if url_next else b'None'
# we've hit a redirect. extract the redirect URL out of the first response header and try again
logger.warning(f"Received a {resp.status_code} from {url_log_safe}, trying to reach redirect url {url_next_log_safe}; attempt #{retries+1}")
logger.warning(
f"Received a {resp.status_code} from {url}, trying to reach redirect url {resp.headers.get('Location', None)}; attempt #{retries+1}"
)
# take the first redirect URL in the response header and try that
url = url_next
url = resp.headers.get("Location", None)
if url is None:
err = f"Webhook notification received redirect to a blank URL from {url_log_safe}. Response headers={resp.headers}"
err = f"Webhook notification received redirect to a blank URL from {url}. Response headers={resp.headers}"
break
else:
# no break condition in the loop encountered; therefore we have hit the maximum number of retries
err = f"Webhook notification max number of retries [{self.MAX_RETRIES}] exceeded. Failed to send webhook notification to {url_log_safe}"
err = f"Webhook notification max number of retries [{self.MAX_RETRIES}] exceeded. Failed to send webhook notification to {url}"
if resp.status_code >= 400:
err = f"Error sending webhook notification: {resp.status_code}"

View File

@@ -19,8 +19,13 @@ class ActivityStreamRegistrar(object):
pre_delete.connect(activity_stream_delete, sender=model, dispatch_uid=str(self.__class__) + str(model) + "_delete")
for m2mfield in model._meta.many_to_many:
m2m_attr = getattr(model, m2mfield.name)
m2m_changed.connect(activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate")
try:
m2m_attr = getattr(model, m2mfield.name)
m2m_changed.connect(
activity_stream_associate, sender=m2m_attr.through, dispatch_uid=str(self.__class__) + str(m2m_attr.through) + "_associate"
)
except AttributeError:
pass
def disconnect(self, model):
if model in self.models:

View File

@@ -48,6 +48,11 @@ class SimpleDAG(object):
'''
self.node_to_edges_by_label = dict()
def __contains__(self, obj):
if self.node['node_object'] in self.node_obj_to_node_index:
return True
return False
def __len__(self):
return len(self.nodes)

View File

@@ -122,11 +122,8 @@ class WorkflowDAG(SimpleDAG):
if not job:
continue
elif job.can_cancel:
cancel_finished = False
job.cancel()
# If the job is not yet in a terminal state after .cancel(),
# the TaskManager still needs to process it.
if job.status not in ('successful', 'failed', 'canceled', 'error'):
cancel_finished = False
return cancel_finished
def is_workflow_done(self):

View File

@@ -196,10 +196,6 @@ class WorkflowManager(TaskBase):
workflow_job.start_args = '' # blank field to remove encrypted passwords
workflow_job.save(update_fields=['status', 'start_args'])
status_changed = True
else:
# Speed-up: schedule the task manager so it can process the
# canceled pending jobs without waiting for the next cycle.
ScheduleTaskManager().schedule()
else:
dnr_nodes = dag.mark_dnr_nodes()
WorkflowJobNode.objects.bulk_update(dnr_nodes, ['do_not_run'])
@@ -241,8 +237,6 @@ class WorkflowManager(TaskBase):
job = spawn_node.unified_job_template.create_unified_job(**kv)
spawn_node.job = job
spawn_node.save()
if spawn_node.ancestor_artifacts and isinstance(spawn_node.unified_job_template, WorkflowJobTemplate):
job.seed_root_ancestor_artifacts(spawn_node.ancestor_artifacts)
logger.debug('Spawned %s in %s for node %s', job.log_format, workflow_job.log_format, spawn_node.pk)
can_start = True
if isinstance(spawn_node.unified_job_template, WorkflowJobTemplate):
@@ -449,29 +443,17 @@ class TaskManager(TaskBase):
self.controlplane_ig = self.tm_models.instance_groups.controlplane_ig
def process_job_dep_failures(self, task):
"""If job depends on a job that has failed or been canceled, mark as failed.
Returns True if a dep failure was found, False otherwise.
"""
"""If job depends on a job that has failed, mark as failed and handle misc stuff."""
for dep in task.dependent_jobs.all():
# if we detect a failed, error, or canceled dependency, go ahead and fail this task.
if dep.status in ("error", "failed", "canceled"):
# if we detect a failed or error dependency, go ahead and fail this task.
if dep.status in ("error", "failed"):
task.status = 'failed'
if dep.status == 'canceled':
logger.warning(f'Previous task canceled, failing task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Canceled: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
ScheduleWorkflowManager().schedule() # speedup for dependency chains in workflow, on workflow cancel
else:
logger.warning(f'Previous task failed, failing task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
logger.warning(f'Previous task failed task: {task.id} dep: {dep.id} task manager')
task.job_explanation = 'Previous Task Failed: {"job_type": "%s", "job_name": "%s", "job_id": "%s"}' % (
get_type_for_model(type(dep)),
dep.name,
dep.id,
)
task.save(update_fields=['status', 'job_explanation'])
task.websocket_emit_status('failed')
self.pre_start_failed.append(task.id)
@@ -563,17 +545,8 @@ class TaskManager(TaskBase):
logger.warning("Task manager has reached time out while processing pending jobs, exiting loop early")
break
if task.cancel_flag:
logger.debug(f"Canceling pending task {task.log_format} because cancel_flag is set")
task.status = 'canceled'
task.job_explanation = gettext_noop("This job was canceled before it started.")
task.save(update_fields=['status', 'job_explanation'])
task.websocket_emit_status('canceled')
self.pre_start_failed.append(task.id)
ScheduleWorkflowManager().schedule()
continue
if self.process_job_dep_failures(task):
has_failed = self.process_job_dep_failures(task)
if has_failed:
continue
blocked_by = self.job_blocked_by(task)

View File

@@ -36,6 +36,7 @@ from awx.main.models import (
Inventory,
InventorySource,
Job,
JobHostSummary,
Organization,
Project,
Role,
@@ -250,9 +251,45 @@ def migrate_children_from_deleted_group_to_parent_groups(sender, **kwargs):
pass
# Host.last_job and Host.last_job_host_summary are now derived from
# JobHostSummary.latest_for_host / latest_job_for_host.
# No signal handlers needed to maintain these denormalized FKs.
# Update host pointers to last_job and last_job_host_summary when a job is deleted
def _update_host_last_jhs(host):
jhs_qs = JobHostSummary.objects.filter(host__pk=host.pk)
try:
jhs = jhs_qs.order_by('-job__pk')[0]
except IndexError:
jhs = None
update_fields = []
try:
last_job = jhs.job if jhs else None
except Job.DoesNotExist:
# The job (and its summaries) have already been/are currently being
# deleted, so there's no need to update the host w/ a reference to it
return
if host.last_job != last_job:
host.last_job = last_job
update_fields.append('last_job')
if host.last_job_host_summary != jhs:
host.last_job_host_summary = jhs
update_fields.append('last_job_host_summary')
if update_fields:
host.save(update_fields=update_fields)
@receiver(pre_delete, sender=Job)
def save_host_pks_before_job_delete(sender, **kwargs):
instance = kwargs['instance']
hosts_qs = Host.objects.filter(last_job__pk=instance.pk)
instance._saved_hosts_pks = set(hosts_qs.values_list('pk', flat=True))
@receiver(post_delete, sender=Job)
def update_host_last_job_after_job_deleted(sender, **kwargs):
instance = kwargs['instance']
hosts_pks = getattr(instance, '_saved_hosts_pks', [])
for host in Host.objects.filter(pk__in=hosts_pks):
_update_host_last_jhs(host)
# Set via ActivityStreamRegistrar to record activity stream events

View File

@@ -54,6 +54,9 @@ def try_load_query_file(artifact_dir) -> Tuple[bool, Optional[dict]]:
returns the contents of ansible_data.json if present
"""
if not flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
return False, None
queries_path = os.path.join(artifact_dir, COLLECTION_FILENAME)
if not os.path.isfile(queries_path):
logger.info(f"no query file found: {queries_path}")
@@ -274,6 +277,20 @@ class RunnerCallback:
def artifacts_handler(self, artifact_dir):
success, query_file_contents = try_load_query_file(artifact_dir)
if success:
self.delay_update(event_queries_processed=False)
collections_info = collect_queries(query_file_contents)
for collection, data in collections_info.items():
version = data['version']
event_query = data['host_query']
instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query)
try:
instance.validate_unique()
instance.save()
logger.info(f"eventy query for collection {collection}, version {version} created")
except ValidationError as e:
logger.info(e)
if 'installed_collections' in query_file_contents:
self.delay_update(installed_collections=query_file_contents['installed_collections'])
else:
@@ -284,21 +301,6 @@ class RunnerCallback:
else:
logger.warning(f'The file {COLLECTION_FILENAME} unexpectedly did not contain ansible_version')
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
self.delay_update(event_queries_processed=False)
collections_info = collect_queries(query_file_contents)
for collection, data in collections_info.items():
version = data['version']
event_query = data['host_query']
instance = EventQuery(fqcn=collection, collection_version=version, event_query=event_query)
try:
instance.validate_unique()
instance.save()
logger.info(f"event query for collection {collection}, version {version} created")
except ValidationError as e:
logger.info(e)
self.artifacts_processed = True

View File

@@ -25,8 +25,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
log_data = log_data or {}
log_data['inventory_id'] = inventory_id
log_data['written_ct'] = 0
# Dict mapping host name -> bool (True if a fact file was written)
hosts_cached = {}
hosts_cached = []
# Create the fact_cache directory inside artifacts_dir
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
@@ -38,14 +37,13 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
last_write_time = None
for host in hosts:
hosts_cached.append(host.name)
if not host.ansible_facts_modified or (timeout and host.ansible_facts_modified < now() - datetime.timedelta(seconds=timeout)):
hosts_cached[host.name] = False
continue # facts are expired - do not write them
filepath = os.path.join(fact_cache_dir, host.name)
if not os.path.realpath(filepath).startswith(fact_cache_dir):
logger.error(f'facts for host {smart_str(host.name)} could not be cached')
hosts_cached[host.name] = False
continue
try:
@@ -53,18 +51,9 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
os.chmod(f.name, 0o600)
json.dump(host.ansible_facts, f)
log_data['written_ct'] += 1
# Backdate the file by 2 seconds so finish_fact_cache can reliably
# distinguish these reference files from files updated by ansible.
# This guarantees fact file mtime < summary file mtime even with
# zipfile's 2-second timestamp rounding during artifact transfer.
mtime = os.path.getmtime(filepath)
backdated = mtime - 2
os.utime(filepath, (backdated, backdated))
last_write_time = backdated
hosts_cached[host.name] = True
last_write_time = os.path.getmtime(filepath)
except IOError:
logger.error(f'facts for host {smart_str(host.name)} could not be cached')
hosts_cached[host.name] = False
continue
# Write summary file directly to the artifacts_dir
@@ -73,6 +62,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
summary_data = {
'last_write_time': last_write_time,
'hosts_cached': hosts_cached,
'written_ct': log_data['written_ct'],
}
with open(summary_file, 'w', encoding='utf-8') as f:
json.dump(summary_data, f, indent=2)
@@ -84,7 +74,7 @@ def start_fact_cache(hosts, artifacts_dir, timeout=None, inventory_id=None, log_
msg='Inventory {inventory_id} host facts: updated {updated_ct}, cleared {cleared_ct}, unchanged {unmodified_ct}, took {delta:.3f} s',
add_log_data=True,
)
def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, job_created=None, log_data=None):
def finish_fact_cache(artifacts_dir, job_id=None, inventory_id=None, log_data=None):
log_data = log_data or {}
log_data['inventory_id'] = inventory_id
log_data['updated_ct'] = 0
@@ -99,118 +89,63 @@ def finish_fact_cache(host_qs, artifacts_dir, job_id=None, inventory_id=None, jo
try:
with open(summary_path, 'r', encoding='utf-8') as f:
summary = json.load(f)
facts_write_time = os.path.getmtime(summary_path)
facts_write_time = os.path.getmtime(summary_path) # After successful read
except (json.JSONDecodeError, OSError) as e:
logger.error(f'Error reading summary file at {summary_path}: {e}')
return
hosts_cached_map = summary.get('hosts_cached', {})
host_names = summary.get('hosts_cached', [])
hosts_cached = Host.objects.filter(name__in=host_names).order_by('id').iterator()
# Path where individual fact files were written
fact_cache_dir = os.path.join(artifacts_dir, 'fact_cache')
hosts_to_update = []
# Phase 1: Scan files on disk to discover which hosts have updated or missing facts
hosts_with_updates = set() # hostnames whose fact file was modified by Ansible
hosts_to_clear = [] # hostnames where Ansible removed the fact file
seen_in_dir = set() # hostnames we found as files on disk
for host in hosts_cached:
filepath = os.path.join(fact_cache_dir, host.name)
if not os.path.realpath(filepath).startswith(fact_cache_dir):
logger.error(f'Invalid path for facts file: {filepath}')
continue
if os.path.isdir(fact_cache_dir):
for filename in os.listdir(fact_cache_dir):
if filename not in hosts_cached_map:
continue # not an expected host for this job
if os.path.exists(filepath):
# If the file changed since we wrote the last facts file, pre-playbook run...
modified = os.path.getmtime(filepath)
if not facts_write_time or modified >= facts_write_time:
try:
with codecs.open(filepath, 'r', encoding='utf-8') as f:
ansible_facts = json.load(f)
except ValueError:
continue
filepath = os.path.join(fact_cache_dir, filename)
if os.path.islink(filepath):
logger.error(f'Invalid path for facts file: {filepath}')
continue
if not os.path.isfile(filepath):
continue
seen_in_dir.add(filename)
try:
modified = os.path.getmtime(filepath)
except OSError as e:
logger.warning(f'Could not stat facts file {filepath}: {e}')
continue
if modified >= facts_write_time:
hosts_with_updates.add(filename)
if ansible_facts != host.ansible_facts:
host.ansible_facts = ansible_facts
host.ansible_facts_modified = now()
hosts_to_update.append(host)
logger.info(
f'New fact for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}',
extra=dict(
inventory_id=host.inventory.id,
host_name=host.name,
ansible_facts=host.ansible_facts,
ansible_facts_modified=host.ansible_facts_modified.isoformat(),
job_id=job_id,
),
)
log_data['updated_ct'] += 1
else:
log_data['unmodified_ct'] += 1
else:
log_data['unmodified_ct'] += 1
# Check for files we wrote pre-job that are now missing (Ansible cleared facts)
for hostname, was_written in hosts_cached_map.items():
if hostname in seen_in_dir:
continue # already handled above
if was_written:
hosts_to_clear.append(hostname)
else:
log_data['unmodified_ct'] += 1
# if the file goes missing, ansible removed it (likely via clear_facts)
# if the file goes missing, but the host has not started facts, then we should not clear the facts
host.ansible_facts = {}
host.ansible_facts_modified = now()
hosts_to_update.append(host)
logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
log_data['cleared_ct'] += 1
# Phase 2: Stream updated facts to database in batches
if hosts_with_updates:
hosts_to_save = []
total_rows_updated = 0
for host in host_qs.filter(name__in=list(hosts_with_updates)).select_related('inventory').iterator():
filepath = os.path.join(fact_cache_dir, host.name)
try:
with codecs.open(filepath, 'r', encoding='utf-8') as f:
new_facts = json.load(f)
except (ValueError, OSError):
continue
if len(hosts_to_update) >= 100:
bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])
hosts_to_update = []
if new_facts != host.ansible_facts:
host.ansible_facts = new_facts
host.ansible_facts_modified = now()
hosts_to_save.append(host)
logger.info(
f'New fact for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}',
extra=dict(
inventory_id=host.inventory.id,
host_name=host.name,
ansible_facts=host.ansible_facts,
ansible_facts_modified=host.ansible_facts_modified.isoformat(),
job_id=job_id,
),
)
log_data['updated_ct'] += 1
else:
log_data['unmodified_ct'] += 1
if len(hosts_to_save) >= 100:
total_rows_updated += bulk_update_sorted_by_id(Host, hosts_to_save, fields=['ansible_facts', 'ansible_facts_modified'])
hosts_to_save = []
if hosts_to_save:
total_rows_updated += bulk_update_sorted_by_id(Host, hosts_to_save, fields=['ansible_facts', 'ansible_facts_modified'])
# Mismatch means a concurrent process changed or deleted hosts between our read and bulk update
if total_rows_updated != log_data['updated_ct']:
logger.warning(
f'Fact update for inventory {inventory_id} job {job_id}: expected to update {log_data["updated_ct"]} hosts but {total_rows_updated} rows were changed'
)
# Phase 3: Clear facts for hosts whose files were removed by Ansible
if hosts_to_clear:
hosts = list(host_qs.filter(name__in=hosts_to_clear).select_related('inventory'))
clear_hosts = []
for host in hosts:
if job_created and host.ansible_facts_modified and host.ansible_facts_modified > job_created:
logger.warning(
f'Skipping fact clear for host {smart_str(host.name)} in job {job_id} '
f'inventory {inventory_id}: host ansible_facts_modified '
f'({host.ansible_facts_modified.isoformat()}) is after this job\'s '
f'created time ({job_created.isoformat()}). '
f'A concurrent job likely updated this host\'s facts while this job was running.'
)
log_data['unmodified_ct'] += 1
else:
host.ansible_facts = {}
host.ansible_facts_modified = now()
clear_hosts.append(host)
logger.info(f'Facts cleared for inventory {smart_str(host.inventory.name)} host {smart_str(host.name)}')
log_data['cleared_ct'] += 1
if clear_hosts:
rows = bulk_update_sorted_by_id(Host, clear_hosts, fields=['ansible_facts', 'ansible_facts_modified'])
if rows != len(clear_hosts):
logger.warning(f'Fact clear for inventory {inventory_id} job {job_id}: expected to clear {len(clear_hosts)} hosts but {rows} rows were changed')
logger.debug(f'Updated {log_data["updated_ct"]} host facts for inventory {inventory_id} in job {job_id}')
bulk_update_sorted_by_id(Host, hosts_to_update, fields=['ansible_facts', 'ansible_facts_modified'])

View File

@@ -17,6 +17,7 @@ import urllib.parse as urlparse
# Django
from django.conf import settings
from django.db import transaction
# Shared code for the AWX platform
from awx_plugins.interfaces._temporary_private_container_api import CONTAINER_ROOT, get_incontainer_path
@@ -83,7 +84,6 @@ from awx.main.utils.common import (
create_partition,
ScheduleWorkflowManager,
ScheduleTaskManager,
getattr_dne,
)
from awx.conf.license import get_license
from awx.main.utils.handlers import SpecialInventoryHandler
@@ -92,90 +92,9 @@ from awx.main.utils.update_model import update_model
# Django flags
from flags.state import flag_enabled
# Workload Identity
from ansible_base.lib.workload_identity.controller import AutomationControllerJobScope
from awx.main.utils.workload_identity import retrieve_workload_identity_jwt_with_claims
logger = logging.getLogger('awx.main.tasks.jobs')
def populate_claims_for_workload(unified_job) -> dict:
"""
Extract JWT claims from a Controller workload for the aap_controller_automation_job scope.
"""
claims = {
AutomationControllerJobScope.CLAIM_JOB_ID: unified_job.id,
AutomationControllerJobScope.CLAIM_JOB_NAME: unified_job.name,
AutomationControllerJobScope.CLAIM_LAUNCH_TYPE: unified_job.launch_type,
}
# Related objects in the UnifiedJob model, applies to all job types
# null cases are omitted because of OIDC
if organization := getattr_dne(unified_job, 'organization'):
claims[AutomationControllerJobScope.CLAIM_ORGANIZATION_NAME] = organization.name
claims[AutomationControllerJobScope.CLAIM_ORGANIZATION_ID] = organization.id
if ujt := getattr_dne(unified_job, 'unified_job_template'):
claims[AutomationControllerJobScope.CLAIM_UNIFIED_JOB_TEMPLATE_NAME] = ujt.name
claims[AutomationControllerJobScope.CLAIM_UNIFIED_JOB_TEMPLATE_ID] = ujt.id
if instance_group := getattr_dne(unified_job, 'instance_group'):
claims[AutomationControllerJobScope.CLAIM_INSTANCE_GROUP_NAME] = instance_group.name
claims[AutomationControllerJobScope.CLAIM_INSTANCE_GROUP_ID] = instance_group.id
# Related objects on concrete models, may not be valid for type of unified_job
if inventory := getattr_dne(unified_job, 'inventory', None):
claims[AutomationControllerJobScope.CLAIM_INVENTORY_NAME] = inventory.name
claims[AutomationControllerJobScope.CLAIM_INVENTORY_ID] = inventory.id
if execution_environment := getattr_dne(unified_job, 'execution_environment', None):
claims[AutomationControllerJobScope.CLAIM_EXECUTION_ENVIRONMENT_NAME] = execution_environment.name
claims[AutomationControllerJobScope.CLAIM_EXECUTION_ENVIRONMENT_ID] = execution_environment.id
if project := getattr_dne(unified_job, 'project', None):
claims[AutomationControllerJobScope.CLAIM_PROJECT_NAME] = project.name
claims[AutomationControllerJobScope.CLAIM_PROJECT_ID] = project.id
if jt := getattr_dne(unified_job, 'job_template', None):
claims[AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_NAME] = jt.name
claims[AutomationControllerJobScope.CLAIM_JOB_TEMPLATE_ID] = jt.id
# Only valid for job templates
if hasattr(unified_job, 'playbook'):
claims[AutomationControllerJobScope.CLAIM_PLAYBOOK_NAME] = unified_job.playbook
# Not valid for inventory updates and system jobs
if hasattr(unified_job, 'job_type'):
claims[AutomationControllerJobScope.CLAIM_JOB_TYPE] = unified_job.job_type
launched_by: dict = unified_job.launched_by
if 'name' in launched_by:
claims[AutomationControllerJobScope.CLAIM_LAUNCHED_BY_NAME] = launched_by['name']
if 'id' in launched_by:
claims[AutomationControllerJobScope.CLAIM_LAUNCHED_BY_ID] = launched_by['id']
return claims
def retrieve_workload_identity_jwt(
unified_job: UnifiedJob,
audience: str,
scope: str,
workload_ttl_seconds: int | None = None,
) -> str:
"""Retrieve JWT token from workload claims.
Raises:
RuntimeError: if the workload identity client is not configured.
"""
return retrieve_workload_identity_jwt_with_claims(
populate_claims_for_workload(unified_job),
audience,
scope,
workload_ttl_seconds,
)
def with_path_cleanup(f):
@functools.wraps(f)
def _wrapped(self, *args, **kwargs):
@@ -202,7 +121,6 @@ def dispatch_waiting_jobs(binder):
if not kwargs:
kwargs = {}
binder.control('run', data={'task': serialize_task(uj._get_task_class()), 'args': [uj.id], 'kwargs': kwargs, 'uuid': uj.celery_task_id})
UnifiedJob.objects.filter(pk=uj.pk, status='waiting').update(status='running', start_args='')
class BaseTask(object):
@@ -217,63 +135,6 @@ class BaseTask(object):
self.update_attempts = int(getattr(settings, 'DISPATCHER_DB_DOWNTOWN_TOLLERANCE', settings.DISPATCHER_DB_DOWNTIME_TOLERANCE) / 5)
self.runner_callback = self.callback_class(model=self.model)
@functools.cached_property
def _credentials(self):
"""
Credentials for the task execution.
Fetches credentials once using build_credentials_list() and stores
them for the duration of the task to avoid redundant database queries.
"""
credentials_list = self.build_credentials_list(self.instance)
# Convert to list to prevent re-evaluation of QuerySet
return list(credentials_list)
def populate_workload_identity_tokens(self, additional_credentials=None):
"""
Populate credentials with workload identity tokens.
Sets the context on Credential objects that have input sources
using compatible external credential types.
"""
credentials = list(self._credentials)
if additional_credentials:
credentials.extend(additional_credentials)
credential_input_sources = (
(credential.context, src)
for credential in credentials
for src in credential.input_sources.all()
if any(
field.get('id') == 'workload_identity_token' and field.get('internal')
for field in src.source_credential.credential_type.inputs.get('fields', [])
)
)
for credential_ctx, input_src in credential_input_sources:
if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
effective_timeout = self.get_instance_timeout(self.instance)
workload_ttl = effective_timeout if effective_timeout else None
try:
jwt = retrieve_workload_identity_jwt(
self.instance,
audience=input_src.source_credential.get_input('url'),
scope=AutomationControllerJobScope.name,
workload_ttl_seconds=workload_ttl,
)
# Store token keyed by input source PK, since a credential can have
# multiple input sources (one per field), each potentially with a different audience
credential_ctx[input_src.pk] = {"workload_identity_token": jwt}
except Exception as e:
self.instance.job_explanation = (
f'Could not generate workload identity token for credential {input_src.source_credential.name} used in this job. Error:\n{e}'
)
self.instance.status = 'error'
self.instance.save()
else:
self.instance.job_explanation = (
f'Flag FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED is not enabled, required for credential {input_src.source_credential.name} used in this job.'
)
self.instance.status = 'error'
self.instance.save()
def update_model(self, pk, _attempt=0, **updates):
return update_model(self.model, pk, _attempt=0, _max_attempts=self.update_attempts, **updates)
@@ -425,19 +286,6 @@ class BaseTask(object):
private_data_files['credentials'][credential] = self.write_private_data_file(private_data_dir, None, data, sub_dir='env')
for credential, data in private_data.get('certificates', {}).items():
self.write_private_data_file(private_data_dir, 'ssh_key_data-cert.pub', data, sub_dir=os.path.join('artifacts', str(self.instance.id)))
# Copy vendor collections to private_data_dir for indirect node counting
# This makes external query files available to the callback plugin in EEs
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
vendor_src = '/var/lib/awx/vendor_collections'
vendor_dest = os.path.join(private_data_dir, 'vendor_collections')
if os.path.exists(vendor_src):
try:
shutil.copytree(vendor_src, vendor_dest)
logger.debug(f"Copied vendor collections from {vendor_src} to {vendor_dest}")
except Exception as e:
logger.warning(f"Failed to copy vendor collections: {e}")
return private_data_files, ssh_key_data
def build_passwords(self, instance, runtime_passwords):
@@ -511,7 +359,6 @@ class BaseTask(object):
return []
def get_instance_timeout(self, instance):
"""Return the effective job timeout in seconds."""
global_timeout_setting_name = instance._global_timeout_setting()
if global_timeout_setting_name:
global_timeout = getattr(settings, global_timeout_setting_name, 0)
@@ -620,32 +467,48 @@ class BaseTask(object):
def should_use_fact_cache(self):
return False
def transition_status(self, pk: int) -> bool:
"""Atomically transition status to running, if False returned, another process got it"""
with transaction.atomic():
# Explanation of parts for the fetch:
# .values - avoid loading a full object, this is known to lead to deadlocks due to signals
# the signals load other related rows which another process may be locking, and happens in practice
# of=('self',) - keeps FK tables out of the lock list, another way deadlocks can happen
# .get - just load the single job
instance_data = UnifiedJob.objects.select_for_update(of=('self',)).values('status', 'cancel_flag').get(pk=pk)
# If status is not waiting (obtained under lock) then this process does not have clearence to run
if instance_data['status'] == 'waiting':
if instance_data['cancel_flag']:
updated_status = 'canceled'
else:
updated_status = 'running'
# Explanation of the update:
# .filter - again, do not load the full object
# .update - a bulk update on just that one row, avoid loading unintended data
UnifiedJob.objects.filter(pk=pk).update(status=updated_status, start_args='')
elif instance_data['status'] == 'running':
logger.info(f'Job {pk} is being ran by another process, exiting')
return False
return True
@with_path_cleanup
@with_signal_handling
def run(self, pk, **kwargs):
"""
Run the job/task and capture its output.
"""
if not self.instance: # Used to skip fetch for local runs
# Load the instance
self.instance = self.update_model(pk)
# status should be "running" from dispatch_waiting_jobs,
# but may still be "waiting" if the worker picked this up before the status update landed.
if self.instance.status == 'waiting':
UnifiedJob.objects.filter(pk=pk).update(status="running", start_args='')
self.instance.refresh_from_db()
if not self.transition_status(pk):
logger.info(f'Job {pk} is being ran by another process, exiting')
return
# Load the instance
self.instance = self.update_model(pk)
if self.instance.status != 'running':
logger.error(f'Not starting {self.instance.status} task pk={pk} because its status "{self.instance.status}" is not expected')
return
if self.instance.cancel_flag:
self.instance = self.update_model(pk, status='canceled')
self.instance.websocket_emit_status('canceled')
return
self.instance.websocket_emit_status("running")
status, rc = 'error', None
self.runner_callback.event_ct = 0
@@ -684,12 +547,6 @@ class BaseTask(object):
if not os.path.exists(settings.AWX_ISOLATION_BASE_PATH):
raise RuntimeError('AWX_ISOLATION_BASE_PATH=%s does not exist' % settings.AWX_ISOLATION_BASE_PATH)
if flag_enabled("FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED"):
logger.info(f'Generating workload identity tokens for {self.instance.log_format}')
self.populate_workload_identity_tokens()
if self.instance.status == 'error':
raise RuntimeError('not starting %s task' % self.instance.status)
# May have to serialize the value
private_data_files, ssh_key_data = self.build_private_data_files(self.instance, private_data_dir)
passwords = self.build_passwords(self.instance, kwargs)
@@ -707,7 +564,7 @@ class BaseTask(object):
self.runner_callback.job_created = str(self.instance.created)
credentials = self._credentials
credentials = self.build_credentials_list(self.instance)
container_root = None
if settings.IS_K8S and isinstance(self.instance, ProjectUpdate):
@@ -1002,29 +859,6 @@ class RunJob(SourceControlMixin, BaseTask):
model = Job
event_model = JobEvent
def _extract_credentials_of_kind(self, kind: str):
return (cred for cred in self._credentials if cred.credential_type.kind == kind)
@property
def _machine_credential(self) -> object:
"""Get machine credential."""
return next(self._extract_credentials_of_kind('ssh'), None)
@property
def _vault_credentials(self) -> list[object]:
"""Get vault credentials."""
return list(self._extract_credentials_of_kind('vault'))
@property
def _network_credentials(self) -> list[object]:
"""Get network credentials."""
return list(self._extract_credentials_of_kind('net'))
@property
def _cloud_credentials(self) -> list[object]:
"""Get cloud credentials."""
return list(self._extract_credentials_of_kind('cloud'))
def build_private_data(self, job, private_data_dir):
"""
Returns a dict of the form
@@ -1042,7 +876,7 @@ class RunJob(SourceControlMixin, BaseTask):
}
"""
private_data = {'credentials': {}}
for credential in self._credentials:
for credential in job.credentials.prefetch_related('input_sources__source_credential').all():
# If we were sent SSH credentials, decrypt them and send them
# back (they will be written to a temporary file).
if credential.has_input('ssh_key_data'):
@@ -1058,14 +892,14 @@ class RunJob(SourceControlMixin, BaseTask):
and ansible-vault.
"""
passwords = super(RunJob, self).build_passwords(job, runtime_passwords)
cred = self._machine_credential
cred = job.machine_credential
if cred:
for field in ('ssh_key_unlock', 'ssh_password', 'become_password', 'vault_password'):
value = runtime_passwords.get(field, cred.get_input('password' if field == 'ssh_password' else field, default=''))
if value not in ('', 'ASK'):
passwords[field] = value
for cred in self._vault_credentials:
for cred in job.vault_credentials:
field = 'vault_password'
vault_id = cred.get_input('vault_id', default=None)
if vault_id:
@@ -1081,7 +915,7 @@ class RunJob(SourceControlMixin, BaseTask):
key unlock over network key unlock.
'''
if 'ssh_key_unlock' not in passwords:
for cred in self._network_credentials:
for cred in job.network_credentials:
if cred.inputs.get('ssh_key_unlock'):
passwords['ssh_key_unlock'] = runtime_passwords.get('ssh_key_unlock', cred.get_input('ssh_key_unlock', default=''))
break
@@ -1116,11 +950,11 @@ class RunJob(SourceControlMixin, BaseTask):
# Set environment variables for cloud credentials.
cred_files = private_data_files.get('credentials', {})
for cloud_cred in self._cloud_credentials:
for cloud_cred in job.cloud_credentials:
if cloud_cred and cloud_cred.credential_type.namespace == 'openstack' and cred_files.get(cloud_cred, ''):
env['OS_CLIENT_CONFIG_FILE'] = get_incontainer_path(cred_files.get(cloud_cred, ''), private_data_dir)
for network_cred in self._network_credentials:
for network_cred in job.network_credentials:
env['ANSIBLE_NET_USERNAME'] = network_cred.get_input('username', default='')
env['ANSIBLE_NET_PASSWORD'] = network_cred.get_input('password', default='')
@@ -1138,11 +972,12 @@ class RunJob(SourceControlMixin, BaseTask):
('ANSIBLE_COLLECTIONS_PATH', 'collections_path', 'requirements_collections', '~/.ansible/collections:/usr/share/ansible/collections'),
]
path_vars.append(
('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'),
)
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
path_vars.append(
('ANSIBLE_CALLBACK_PLUGINS', 'callback_plugins', 'plugins_path', '~/.ansible/plugins:/plugins/callback:/usr/share/ansible/plugins/callback'),
)
config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)) + ['callbacks_enabled'])
config_values = read_ansible_config(os.path.join(private_data_dir, 'project'), list(map(lambda x: x[1], path_vars)))
for env_key, config_setting, folder, default in path_vars:
paths = default.split(':')
@@ -1157,16 +992,10 @@ class RunJob(SourceControlMixin, BaseTask):
paths = [os.path.join(CONTAINER_ROOT, folder)] + paths
env[env_key] = os.pathsep.join(paths)
env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count'
if 'callbacks_enabled' in config_values:
env['ANSIBLE_CALLBACKS_ENABLED'] += ',' + config_values['callbacks_enabled']
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
env['AWX_COLLECT_HOST_QUERIES'] = '1'
# Add vendor collections path for external query file discovery
vendor_collections_path = os.path.join(CONTAINER_ROOT, 'vendor_collections')
env['ANSIBLE_COLLECTIONS_PATH'] = f"{vendor_collections_path}:{env['ANSIBLE_COLLECTIONS_PATH']}"
logger.debug(f"ANSIBLE_COLLECTIONS_PATH updated for vendor collections: {env['ANSIBLE_COLLECTIONS_PATH']}")
env['ANSIBLE_CALLBACKS_ENABLED'] = 'indirect_instance_count'
if 'callbacks_enabled' in config_values:
env['ANSIBLE_CALLBACKS_ENABLED'] += ':' + config_values['callbacks_enabled']
return env
@@ -1175,7 +1004,7 @@ class RunJob(SourceControlMixin, BaseTask):
Build command line argument list for running ansible-playbook,
optionally using ssh-agent for public/private key authentication.
"""
creds = self._machine_credential
creds = job.machine_credential
ssh_username, become_username, become_method = '', '', ''
if creds:
@@ -1327,17 +1156,10 @@ class RunJob(SourceControlMixin, BaseTask):
return
if self.should_use_fact_cache() and self.runner_callback.artifacts_processed:
job.log_lifecycle("finish_job_fact_cache")
if job.inventory.kind == 'constructed':
hosts_qs = job.get_source_hosts_for_constructed_inventory()
else:
hosts_qs = job.inventory.hosts
hosts_qs = hosts_qs.only(*HOST_FACTS_FIELDS)
finish_fact_cache(
hosts_qs,
artifacts_dir=os.path.join(private_data_dir, 'artifacts', str(job.id)),
job_id=job.id,
inventory_id=job.inventory_id,
job_created=job.created,
)
def final_run_hook(self, job, status, private_data_dir):
@@ -1506,6 +1328,7 @@ class RunProjectUpdate(BaseTask):
'local_path': os.path.basename(project_update.project.local_path),
'project_path': project_update.get_project_path(check_if_exists=False), # deprecated
'insights_url': settings.INSIGHTS_URL_BASE,
'oidc_endpoint': settings.INSIGHTS_OIDC_ENDPOINT,
'awx_license_type': get_license().get('license_type', 'UNLICENSED'),
'awx_version': get_awx_version(),
'scm_url': scm_url,
@@ -1612,14 +1435,16 @@ class RunProjectUpdate(BaseTask):
shutil.copytree(cache_subpath, dest_subpath, symlinks=True)
logger.debug('{0} {1} prepared {2} from cache'.format(type(project).__name__, project.pk, dest_subpath))
pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path')
if not os.path.exists(pdd_plugins_path):
os.mkdir(pdd_plugins_path)
from awx.playbooks import library
if flag_enabled("FEATURE_INDIRECT_NODE_COUNTING_ENABLED"):
# copy the special callback (not stdout type) plugin to get list of collections
pdd_plugins_path = os.path.join(job_private_data_dir, 'plugins_path')
if not os.path.exists(pdd_plugins_path):
os.mkdir(pdd_plugins_path)
from awx.playbooks import library
plugin_file_source = os.path.join(library.__path__[0], 'indirect_instance_count.py')
plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py')
shutil.copyfile(plugin_file_source, plugin_file_dest)
plugin_file_source = os.path.join(library.__path__._path[0], 'indirect_instance_count.py')
plugin_file_dest = os.path.join(pdd_plugins_path, 'indirect_instance_count.py')
shutil.copyfile(plugin_file_source, plugin_file_dest)
def post_run_hook(self, instance, status):
super(RunProjectUpdate, self).post_run_hook(instance, status)
@@ -1682,7 +1507,7 @@ class RunProjectUpdate(BaseTask):
return params
def build_credentials_list(self, project_update):
if project_update.credential:
if project_update.scm_type == 'insights' and project_update.credential:
return [project_update.credential]
return []
@@ -1865,24 +1690,6 @@ class RunInventoryUpdate(SourceControlMixin, BaseTask):
# All credentials not used by inventory source injector
return inventory_update.get_extra_credentials()
def populate_workload_identity_tokens(self, additional_credentials=None):
"""Also generate OIDC tokens for the cloud credential.
The cloud credential is not in _credentials (it is handled by the
inventory source injector), but it may still need a workload identity
token generated for it.
"""
cloud_cred = self.instance.get_cloud_credential()
creds = list(additional_credentials or [])
if cloud_cred:
creds.append(cloud_cred)
super().populate_workload_identity_tokens(additional_credentials=creds or None)
# Override get_cloud_credential on this instance so the injector
# uses the credential with OIDC context instead of doing a fresh
# DB fetch that would lose it.
if cloud_cred and cloud_cred.context:
self.instance.get_cloud_credential = lambda: cloud_cred
def build_project_dir(self, inventory_update, private_data_dir):
source_project = None
if inventory_update.inventory_source:

View File

@@ -393,9 +393,9 @@ def evaluate_policy(instance):
raise PolicyEvaluationError(_('Following certificate settings are missing for OPA_AUTH_TYPE=Certificate: {}').format(cert_settings_missing))
query_paths = [
('Organization', instance.organization.opa_query_path if instance.organization else None),
('Inventory', instance.inventory.opa_query_path if instance.inventory else None),
('Job template', instance.job_template.opa_query_path if instance.job_template else None),
('Organization', instance.organization.opa_query_path),
('Inventory', instance.inventory.opa_query_path),
('Job template', instance.job_template.opa_query_path),
]
violations = dict()
errors = dict()

View File

@@ -69,7 +69,7 @@ def signal_callback():
def with_signal_handling(f):
"""
Change signal handling to make signal_callback return True in event of SIGTERM, SIGINT, or SIGUSR1.
Change signal handling to make signal_callback return True in event of SIGTERM or SIGINT.
"""
@functools.wraps(f)

View File

@@ -93,10 +93,7 @@ def _run_dispatch_startup_common():
# TODO: Enable this on VM installs
if settings.IS_K8S:
try:
write_receptor_config()
except Exception:
logger.exception("Failed to write receptor config, skipping.")
write_receptor_config()
try:
convert_jsonfields()
@@ -760,16 +757,14 @@ def _heartbeat_check_versions(this_inst, instance_list):
def _heartbeat_handle_lost_instances(lost_instances, this_inst):
"""Handle lost instances by reaping their running jobs and marking them offline."""
"""Handle lost instances by reaping their jobs and marking them offline."""
for other_inst in lost_instances:
try:
# Any jobs marked as running will be marked as error
explanation = "Job reaped due to instance shutdown"
reaper.reap(other_inst, job_explanation=explanation)
# Any jobs that were waiting to be processed by this node will be handed back to task manager
UnifiedJob.objects.filter(status='waiting', controller_node=other_inst.hostname).update(status='pending', controller_node='', execution_node='')
reaper.reap_waiting(other_inst, grace_period=0, job_explanation=explanation)
except Exception:
logger.exception('failed to re-process jobs for lost instance {}'.format(other_inst.hostname))
logger.exception('failed to reap jobs for {}'.format(other_inst.hostname))
try:
if settings.AWX_AUTO_DEPROVISION_INSTANCES and other_inst.node_type == "control":
deprovision_hostname = other_inst.hostname

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 1.0.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection v1.5.0. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 1.5.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,19 +0,0 @@
---
authors:
- AWX Project Contributors <awx-project@googlegroups.com>
dependencies: {}
description: External query testing collection v3.0.0. No embedded query file. Not for use in production.
documentation: https://github.com/ansible/awx
homepage: https://github.com/ansible/awx
issues: https://github.com/ansible/awx
license:
- GPL-3.0-or-later
name: external
namespace: demo
readme: README.md
repository: https://github.com/ansible/awx
tags:
- demo
- testing
- external_query
version: 3.0.0

View File

@@ -1,78 +0,0 @@
#!/usr/bin/python
# Same licensing as AWX
from __future__ import absolute_import, division, print_function
__metaclass__ = type
DOCUMENTATION = r'''
---
module: example
short_description: Module for specific live tests
version_added: "2.0.0"
description: This module is part of a test collection in local source. Used for external query testing.
options:
host_name:
description: Name to return as the host name.
required: false
type: str
author:
- AWX Live Tests
'''
EXAMPLES = r'''
- name: Test with defaults
demo.external.example:
- name: Test with custom host name
demo.external.example:
host_name: foo_host
'''
RETURN = r'''
direct_host_name:
description: The name of the host, this will be collected with the feature.
type: str
returned: always
sample: 'foo_host'
'''
from ansible.module_utils.basic import AnsibleModule
def run_module():
module_args = dict(
host_name=dict(type='str', required=False, default='foo_host_default'),
)
result = dict(
changed=False,
other_data='sample_string',
)
module = AnsibleModule(argument_spec=module_args, supports_check_mode=True)
if module.check_mode:
module.exit_json(**result)
result['direct_host_name'] = module.params['host_name']
result['nested_host_name'] = {'host_name': module.params['host_name']}
result['name'] = 'vm-foo'
# non-cononical facts
result['device_type'] = 'Fake Host'
module.exit_json(**result)
def main():
run_module()
if __name__ == '__main__':
main()

View File

@@ -1,11 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- name: Set artifacts via set_stats
ansible.builtin.set_stats:
data: "{{ stats_data }}"
per_host: false
aggregate: false
when: stats_data is defined

View File

@@ -1,21 +0,0 @@
---
# Generated by Claude Opus 4.6 (claude-opus-4-6).
- hosts: all
vars:
extra_value: ""
gather_facts: false
connection: local
tasks:
- name: set a custom fact
set_fact:
foo: "bar{{ extra_value }}"
bar:
a:
b:
- "c"
- "d"
cacheable: true
- name: sleep to create overlap window for concurrent job testing
wait_for:
timeout: 2

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v1_0_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v1_5_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -1,5 +0,0 @@
---
collections:
- name: 'file:///tmp/live_tests/host_query_external_v3_0_0'
type: git
version: devel

View File

@@ -1,8 +0,0 @@
---
- hosts: all
gather_facts: false
connection: local
tasks:
- demo.external.example:
register: result
- debug: var=result

View File

@@ -74,9 +74,9 @@ def temp_analytic_tar():
@pytest.fixture
def mock_analytic_post():
# Patch get_or_generate_candlepin_certificate to skip mTLS path
with mock.patch('awx.main.analytics.core.get_or_generate_candlepin_certificate', return_value=(None, None)):
yield
# Patch the Session.post method to return a mock response with status_code 200
with mock.patch('awx.main.analytics.core.requests.Session.post', return_value=mock.Mock(status_code=200)) as mock_post:
yield mock_post
@pytest.mark.parametrize(
@@ -141,22 +141,15 @@ def mock_analytic_post():
)
@pytest.mark.django_db
def test_ship_credential(setting_map, expected_result, expected_auth, temp_analytic_tar, mock_analytic_post):
with override_settings(**setting_map, AUTOMATION_ANALYTICS_URL='https://example.com/api'):
with mock.patch('awx.main.analytics.core.OIDCClient') as mock_oidc:
mock_oidc_instance = mock.Mock()
mock_oidc_instance.make_request.return_value = mock.Mock(status_code=200)
mock_oidc.return_value = mock_oidc_instance
with override_settings(**setting_map):
result = ship(temp_analytic_tar)
result = ship(temp_analytic_tar)
assert result == expected_result
if expected_auth:
# Verify OIDC client was instantiated with correct credentials
mock_oidc.assert_called_once_with(expected_auth[0], expected_auth[1])
mock_oidc_instance.make_request.assert_called_once()
else:
# When credentials are missing, OIDCClient should not be called
mock_oidc.assert_not_called()
assert result == expected_result
if expected_auth:
mock_analytic_post.assert_called_once()
assert mock_analytic_post.call_args[1]['auth'] == expected_auth
else:
mock_analytic_post.assert_not_called()
@pytest.mark.django_db

View File

@@ -1,11 +1,8 @@
import pytest
from django.test import RequestFactory
from prometheus_client.parser import text_string_to_metric_families
from rest_framework.request import Request
from awx.main import models
from awx.main.analytics.metrics import metrics
from awx.main.analytics.dispatcherd_metrics import get_dispatcherd_metrics
from awx.api.versioning import reverse
EXPECTED_VALUES = {
@@ -80,55 +77,3 @@ def test_metrics_http_methods(get, post, patch, put, options, admin):
assert patch(get_metrics_view_db_only(), user=admin).status_code == 405
assert post(get_metrics_view_db_only(), user=admin).status_code == 405
assert options(get_metrics_view_db_only(), user=admin).status_code == 200
class DummyMetricsResponse:
def __init__(self, payload):
self._payload = payload
def read(self):
return self._payload
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def test_dispatcherd_metrics_node_filter_match(mocker, settings):
settings.CLUSTER_HOST_ID = "awx-1"
payload = b'# HELP test_metric A test metric\n# TYPE test_metric gauge\ntest_metric 1\n'
def fake_urlopen(url, timeout=1.0):
return DummyMetricsResponse(payload)
mocker.patch('urllib.request.urlopen', fake_urlopen)
request = Request(RequestFactory().get('/api/v2/metrics/', {'node': 'awx-1'}))
assert get_dispatcherd_metrics(request) == payload.decode('utf-8')
def test_dispatcherd_metrics_node_filter_excludes_local(mocker, settings):
settings.CLUSTER_HOST_ID = "awx-1"
def fake_urlopen(*args, **kwargs):
raise AssertionError("urlopen should not be called when node filter excludes local node")
mocker.patch('urllib.request.urlopen', fake_urlopen)
request = Request(RequestFactory().get('/api/v2/metrics/', {'node': 'awx-2'}))
assert get_dispatcherd_metrics(request) == ''
def test_dispatcherd_metrics_metric_filter_excludes_unrelated(mocker):
def fake_urlopen(*args, **kwargs):
raise AssertionError("urlopen should not be called when metric filter excludes dispatcherd metrics")
mocker.patch('urllib.request.urlopen', fake_urlopen)
request = Request(RequestFactory().get('/api/v2/metrics/', {'metric': 'awx_system_info'}))
assert get_dispatcherd_metrics(request) == ''

View File

@@ -1,4 +1,3 @@
import os
import pytest
import requests
from unittest import mock
@@ -258,92 +257,3 @@ class TestAnalyticsGenericView:
else:
# assert mock_base_auth_request not called
mock_base_auth_request.assert_not_called()
@pytest.mark.django_db
def test__send_to_analytics_respects_proxy_env_oidc(self):
settings_map = {
'INSIGHTS_TRACKING_STATE': True,
'AUTOMATION_ANALYTICS_URL': 'https://example.com',
'REDHAT_USERNAME': 'redhat_user',
'REDHAT_PASSWORD': 'redhat_pass',
'SUBSCRIPTIONS_CLIENT_ID': '',
'SUBSCRIPTIONS_CLIENT_SECRET': '',
'AWX_TASK_ENV': {'HTTPS_PROXY': '192.168.50.100:1234', 'HTTP_PROXY': '192.168.50.100:5678'},
}
with override_settings(**settings_map):
request = RequestFactory().post('/some/path')
view = AnalyticsGenericView()
with mock.patch('awx.api.views.analytics.OIDCClient') as mock_oidc_client:
mock_client_instance = mock.Mock()
mock_oidc_client.return_value = mock_client_instance
def _check_env_and_respond(*args, **kwargs):
assert os.environ.get('HTTPS_PROXY') == '192.168.50.100:1234'
assert os.environ.get('HTTP_PROXY') == '192.168.50.100:5678'
return mock.Mock(status_code=200)
mock_client_instance.make_request.side_effect = _check_env_and_respond
response = view._send_to_analytics(request, 'POST')
assert response.status_code == 200
mock_client_instance.make_request.assert_called_once()
@pytest.mark.django_db
def test__send_to_analytics_respects_proxy_env_basic_auth(self):
settings_map = {
'INSIGHTS_TRACKING_STATE': True,
'AUTOMATION_ANALYTICS_URL': 'https://example.com',
'REDHAT_USERNAME': 'redhat_user',
'REDHAT_PASSWORD': 'redhat_pass',
'SUBSCRIPTIONS_CLIENT_ID': '',
'SUBSCRIPTIONS_CLIENT_SECRET': '',
'AWX_TASK_ENV': {'HTTPS_PROXY': '192.168.50.100:1234'},
}
with override_settings(**settings_map):
request = RequestFactory().post('/some/path')
view = AnalyticsGenericView()
with mock.patch('awx.api.views.analytics.OIDCClient') as mock_oidc_client, mock.patch(
'awx.api.views.analytics.AnalyticsGenericView._base_auth_request'
) as mock_base_auth:
mock_client_instance = mock.Mock()
mock_oidc_client.return_value = mock_client_instance
mock_client_instance.make_request.side_effect = requests.RequestException("OIDC failed")
def _check_env_and_respond(*args, **kwargs):
assert os.environ.get('HTTPS_PROXY') == '192.168.50.100:1234'
return mock.Mock(status_code=200)
mock_base_auth.side_effect = _check_env_and_respond
response = view._send_to_analytics(request, 'POST')
assert response.status_code == 200
mock_base_auth.assert_called_once()
@pytest.mark.django_db
def test__send_to_analytics_restores_env_after_request(self):
original_value = os.environ.pop('HTTPS_PROXY', None)
settings_map = {
'INSIGHTS_TRACKING_STATE': True,
'AUTOMATION_ANALYTICS_URL': 'https://example.com',
'REDHAT_USERNAME': 'redhat_user',
'REDHAT_PASSWORD': 'redhat_pass',
'SUBSCRIPTIONS_CLIENT_ID': '',
'SUBSCRIPTIONS_CLIENT_SECRET': '',
'AWX_TASK_ENV': {'HTTPS_PROXY': '192.168.50.100:1234'},
}
try:
with override_settings(**settings_map):
request = RequestFactory().post('/some/path')
view = AnalyticsGenericView()
with mock.patch('awx.api.views.analytics.OIDCClient') as mock_oidc_client:
mock_client_instance = mock.Mock()
mock_oidc_client.return_value = mock_client_instance
mock_client_instance.make_request.return_value = mock.Mock(status_code=200)
view._send_to_analytics(request, 'POST')
assert 'HTTPS_PROXY' not in os.environ
finally:
if original_value is not None:
os.environ['HTTPS_PROXY'] = original_value

View File

@@ -1,84 +0,0 @@
import pytest
from awx.api.versioning import reverse
from rest_framework import status
from awx.main.models.jobs import JobTemplate
@pytest.mark.django_db
class TestConfigEndpointFields:
def test_base_fields_all_users(self, get, rando):
url = reverse('api:api_v2_config_view')
response = get(url, rando, expect=200)
assert 'time_zone' in response.data
assert 'license_info' in response.data
assert 'version' in response.data
assert 'eula' in response.data
assert 'analytics_status' in response.data
assert 'analytics_collectors' in response.data
assert 'become_methods' in response.data
@pytest.mark.parametrize(
"role_type",
[
"superuser",
"system_auditor",
"org_admin",
"org_auditor",
"org_project_admin",
],
)
def test_privileged_users_conditional_fields(self, get, user, organization, admin, role_type):
url = reverse('api:api_v2_config_view')
if role_type == "superuser":
test_user = admin
elif role_type == "system_auditor":
test_user = user('system-auditor', is_superuser=False)
test_user.is_system_auditor = True
test_user.save()
elif role_type == "org_admin":
test_user = user('org-admin', is_superuser=False)
organization.admin_role.members.add(test_user)
elif role_type == "org_auditor":
test_user = user('org-auditor', is_superuser=False)
organization.auditor_role.members.add(test_user)
elif role_type == "org_project_admin":
test_user = user('org-project-admin', is_superuser=False)
organization.project_admin_role.members.add(test_user)
response = get(url, test_user, expect=200)
assert 'project_base_dir' in response.data
assert 'project_local_paths' in response.data
assert 'custom_virtualenvs' in response.data
def test_job_template_admin_gets_venvs_only(self, get, user, organization, project, inventory):
"""Test that JobTemplate admin without org access gets only custom_virtualenvs"""
jt_admin = user('jt-admin', is_superuser=False)
jt = JobTemplate.objects.create(name='test-jt', organization=organization, project=project, inventory=inventory)
jt.admin_role.members.add(jt_admin)
url = reverse('api:api_v2_config_view')
response = get(url, jt_admin, expect=200)
assert 'custom_virtualenvs' in response.data
assert 'project_base_dir' not in response.data
assert 'project_local_paths' not in response.data
def test_normal_user_no_conditional_fields(self, get, rando):
url = reverse('api:api_v2_config_view')
response = get(url, rando, expect=200)
assert 'project_base_dir' not in response.data
assert 'project_local_paths' not in response.data
assert 'custom_virtualenvs' not in response.data
def test_unauthenticated_denied(self, get):
"""Test that unauthenticated requests are denied"""
url = reverse('api:api_v2_config_view')
response = get(url, None, expect=401)
assert response.status_code == status.HTTP_401_UNAUTHORIZED

View File

@@ -200,7 +200,6 @@ def test_grant_org_credential_to_org_user_through_user_roles(post, credential, o
@pytest.mark.django_db
def test_grant_org_credential_to_non_org_user_through_role_users(post, credential, organization, org_admin, alice):
# NOTE: this endpoint is going away soon
credential.organization = organization
credential.save()
response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': alice.id}, org_admin)
@@ -209,7 +208,6 @@ def test_grant_org_credential_to_non_org_user_through_role_users(post, credentia
@pytest.mark.django_db
def test_grant_org_credential_to_non_org_user_through_user_roles(post, credential, organization, org_admin, alice):
# NOTE: this endpoint is going away soon
credential.organization = organization
credential.save()
response = post(reverse('api:user_roles_list', kwargs={'pk': alice.id}), {'id': credential.use_role.id}, org_admin)
@@ -218,18 +216,18 @@ def test_grant_org_credential_to_non_org_user_through_user_roles(post, credentia
@pytest.mark.django_db
def test_grant_private_credential_to_user_through_role_users(post, credential, alice, bob):
# NOTE: this endpoint is going away soon
# normal users can't do this
credential.admin_role.members.add(alice)
response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': bob.id}, alice)
assert response.status_code == 403
assert response.status_code == 400
@pytest.mark.django_db
def test_grant_private_credential_to_org_user_through_role_users(post, credential, org_admin, org_member):
# NOTE: this endpoint is going away soon
# org admins can't either
credential.admin_role.members.add(org_admin)
response = post(reverse('api:role_users_list', kwargs={'pk': credential.use_role.id}), {'id': org_member.id}, org_admin)
assert response.status_code == 204
assert response.status_code == 400
@pytest.mark.django_db
@@ -241,18 +239,18 @@ def test_sa_grant_private_credential_to_user_through_role_users(post, credential
@pytest.mark.django_db
def test_grant_private_credential_to_user_through_user_roles(post, credential, alice, bob):
# NOTE: this endpoint is going away soon
# normal users can't do this
credential.admin_role.members.add(alice)
response = post(reverse('api:user_roles_list', kwargs={'pk': bob.id}), {'id': credential.use_role.id}, alice)
assert response.status_code == 403
assert response.status_code == 400
@pytest.mark.django_db
def test_grant_private_credential_to_org_user_through_user_roles(post, credential, org_admin, org_member):
# NOTE: this endpoint is going away soon
# org admins can't either
credential.admin_role.members.add(org_admin)
response = post(reverse('api:user_roles_list', kwargs={'pk': org_member.id}), {'id': credential.use_role.id}, org_admin)
assert response.status_code == 204
assert response.status_code == 400
@pytest.mark.django_db
@@ -284,14 +282,14 @@ def test_grant_org_credential_to_team_through_team_roles(post, credential, organ
@pytest.mark.django_db
def test_sa_grant_private_credential_to_team_through_role_teams(post, credential, admin, team):
# NOTE: this endpoint is going away soon
# not even a system admin can grant a private cred to a team though
response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, admin)
assert response.status_code == 204
assert response.status_code == 400
@pytest.mark.django_db
def test_grant_credential_to_team_different_organization_through_role_teams(post, get, credential, organizations, admin, org_admin, team, team_member):
# NOTE: this endpoint is going away soon
# # Test that credential from different org can be assigned to team by a superuser through role_teams_list endpoint
orgs = organizations(2)
credential.organization = orgs[0]
credential.save()
@@ -301,7 +299,10 @@ def test_grant_credential_to_team_different_organization_through_role_teams(post
# Non-superuser (org_admin) trying cross-org assignment should be denied
response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, org_admin)
assert response.status_code == 400
assert "You cannot grant credential access to a Team not in the credentials' organization" in str(response.data['detail'])
assert (
"You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
in response.data['msg']
)
# Superuser (admin) can do cross-org assignment
response = post(reverse('api:role_teams_list', kwargs={'pk': credential.use_role.id}), {'id': team.id}, admin)
@@ -315,17 +316,20 @@ def test_grant_credential_to_team_different_organization_through_role_teams(post
@pytest.mark.django_db
def test_grant_credential_to_team_different_organization(post, get, credential, organizations, admin, org_admin, team, team_member):
# NOTE: this endpoint is going away soon
# Test that credential from different org can be assigned to team by a superuser
orgs = organizations(2)
credential.organization = orgs[0]
credential.save()
team.organization = orgs[1]
team.save()
# Non-superuser (org_admin) trying cross-org assignment should be denied
# Non-superuser (org_admin, ...) trying cross-org assignment should be denied
response = post(reverse('api:team_roles_list', kwargs={'pk': team.id}), {'id': credential.use_role.id}, org_admin)
assert response.status_code == 400
assert "You cannot grant credential access to a Team not in the credentials' organization" in str(response.data['detail'])
assert (
"You cannot grant a team access to a credential in a different organization. Only superusers can grant cross-organization credential access to teams"
in response.data['msg']
)
# Superuser (system admin) can do cross-org assignment
response = post(reverse('api:team_roles_list', kwargs={'pk': team.id}), {'id': credential.use_role.id}, admin)

View File

@@ -1,7 +1,5 @@
import pytest
from ansible_base.lib.testing.util import feature_flag_enabled, feature_flag_disabled
from awx.main.models import CredentialInputSource
from awx.api.versioning import reverse
@@ -318,60 +316,3 @@ def test_create_credential_input_source_with_already_used_input_returns_400(post
]
all_responses = [post(list_url, params, admin) for params in all_params]
assert all_responses.pop().status_code == 400
@pytest.mark.django_db
def test_credential_input_source_passes_workload_identity_token_when_flag_enabled(vault_credential, external_credential, mocker):
"""Test that workload_identity_token is passed to backend when flag is enabled."""
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
# Add workload_identity_token as an internal field on the external credential type
# so get_input_value resolves it from the per-input-source context
external_credential.credential_type.inputs['fields'].append(
{'id': 'workload_identity_token', 'label': 'Workload Identity Token', 'type': 'string', 'internal': True}
)
# Create an input source
input_source = CredentialInputSource.objects.create(
target_credential=vault_credential,
source_credential=external_credential,
input_field_name='vault_password',
metadata={'key': 'test_key'},
)
# Mock the credential plugin backend
mock_backend = mocker.patch.object(external_credential.credential_type.plugin, 'backend', autospec=True, return_value='test_value')
# Call with context keyed by input source PK
test_context = {input_source.pk: {'workload_identity_token': 'jwt_token_here'}}
result = input_source.get_input_value(context=test_context)
# Verify backend was called with workload_identity_token
assert result == 'test_value'
call_kwargs = mock_backend.call_args[1]
assert call_kwargs['workload_identity_token'] == 'jwt_token_here'
assert call_kwargs['key'] == 'test_key'
@pytest.mark.django_db
def test_credential_input_source_skips_workload_identity_token_when_flag_disabled(vault_credential, external_credential, mocker):
"""Test that workload_identity_token is NOT passed when flag is disabled."""
with feature_flag_disabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
# Create an input source
input_source = CredentialInputSource.objects.create(
target_credential=vault_credential,
source_credential=external_credential,
input_field_name='vault_password',
metadata={'key': 'test_key'},
)
# Mock the credential plugin backend
mock_backend = mocker.patch.object(external_credential.credential_type.plugin, 'backend', autospec=True, return_value='test_value')
# Call with context containing workload_identity_token but NO internal field defined,
# simulating a flag-disabled scenario where tokens are not generated upstream
test_context = {input_source.pk: {'workload_identity_token': 'jwt_token_here'}}
result = input_source.get_input_value(context=test_context)
# Verify backend was called WITHOUT workload_identity_token since the credential type
# does not define it as an internal field (flag-disabled path doesn't register it)
assert result == 'test_value'
call_kwargs = mock_backend.call_args[1]
assert 'workload_identity_token' not in call_kwargs
assert call_kwargs['key'] == 'test_key'

View File

@@ -2,7 +2,6 @@ import json
import pytest
from ansible_base.lib.testing.util import feature_flag_enabled
from awx.main.models.credential import CredentialType, Credential
from awx.api.versioning import reverse
@@ -160,8 +159,7 @@ def test_create_as_admin(get, post, admin):
response = get(reverse('api:credential_type_list'), admin)
assert response.data['count'] == 1
assert response.data['results'][0]['name'] == 'Custom Credential Type'
# Serializer normalizes empty inputs to {'fields': []}
assert response.data['results'][0]['inputs'] == {'fields': []}
assert response.data['results'][0]['inputs'] == {}
assert response.data['results'][0]['injectors'] == {}
assert response.data['results'][0]['managed'] is False
@@ -476,98 +474,3 @@ def test_credential_type_rbac_external_test(post, alice, admin, credentialtype_e
data = {'inputs': {}, 'metadata': {}}
assert post(url, data, admin).status_code == 202
assert post(url, data, alice).status_code == 403
# --- Tests for internal field filtering with None/invalid inputs ---
@pytest.mark.django_db
def test_credential_type_with_none_inputs(get, admin):
"""Test that credential type with empty inputs dict works correctly."""
# Create a credential type with empty dict
ct = CredentialType.objects.create(
kind='cloud',
name='Test Type',
managed=False,
inputs={}, # Empty dict, not None (DB has NOT NULL constraint)
)
url = reverse('api:credential_type_detail', kwargs={'pk': ct.pk})
response = get(url, admin)
assert response.status_code == 200
# Should have normalized inputs to empty dict
assert 'inputs' in response.data
assert isinstance(response.data['inputs'], dict)
assert response.data['inputs']['fields'] == []
@pytest.mark.django_db
def test_credential_type_with_invalid_inputs_type(get, admin):
"""Test that credential type with non-dict inputs doesn't cause errors."""
# Create a credential type with invalid inputs type
ct = CredentialType.objects.create(kind='cloud', name='Test Type', managed=False, inputs={'fields': 'not-a-list'})
url = reverse('api:credential_type_detail', kwargs={'pk': ct.pk})
response = get(url, admin)
assert response.status_code == 200
# Should gracefully handle invalid fields type
assert 'inputs' in response.data
assert response.data['inputs']['fields'] == []
@pytest.mark.django_db
def test_credential_type_filters_internal_fields(get, admin):
"""Test that internal fields are filtered from API responses."""
ct = CredentialType.objects.create(
kind='cloud',
name='Test OIDC Type',
managed=False,
inputs={
'fields': [
{'id': 'url', 'label': 'URL', 'type': 'string'},
{'id': 'token', 'label': 'Token', 'type': 'string', 'secret': True, 'internal': True},
{'id': 'public_field', 'label': 'Public', 'type': 'string'},
]
},
)
url = reverse('api:credential_type_detail', kwargs={'pk': ct.pk})
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
response = get(url, admin)
assert response.status_code == 200
field_ids = [f['id'] for f in response.data['inputs']['fields']]
# Internal field should be filtered out
assert 'token' not in field_ids
assert 'url' in field_ids
assert 'public_field' in field_ids
@pytest.mark.django_db
def test_credential_type_list_filters_internal_fields(get, admin):
"""Test that internal fields are filtered in list view."""
CredentialType.objects.create(
kind='cloud',
name='Test OIDC Type',
managed=False,
inputs={
'fields': [
{'id': 'url', 'label': 'URL', 'type': 'string'},
{'id': 'workload_identity_token', 'label': 'Token', 'type': 'string', 'secret': True, 'internal': True},
]
},
)
url = reverse('api:credential_type_list')
with feature_flag_enabled('FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED'):
response = get(url, admin)
assert response.status_code == 200
# Find our credential type in the results
test_ct = next((ct for ct in response.data['results'] if ct['name'] == 'Test OIDC Type'), None)
assert test_ct is not None
field_ids = [f['id'] for f in test_ct['inputs']['fields']]
# Internal field should be filtered out
assert 'workload_identity_token' not in field_ids
assert 'url' in field_ids

View File

@@ -1,34 +0,0 @@
import pytest
from awx.api.versioning import reverse
from awx.main.models import Host, Inventory
@pytest.mark.django_db
def test_dashboard_hosts_total_excludes_constructed(get, admin_user, organization):
"""
Constructed inventory hosts are not counted in the dashboard
"""
source_inv = Inventory.objects.create(name='source-inv', organization=organization)
source_host = source_inv.hosts.create(name='host1')
constructed = Inventory.objects.create(name='constructed-inv', kind='constructed', organization=organization)
Host.objects.create(name='host1', inventory=constructed, instance_id=str(source_host.pk))
response = get(reverse('api:dashboard_view'), user=admin_user, expect=200)
assert response.data['hosts']['total'] == 1
@pytest.mark.django_db
def test_host_list_still_returns_constructed(get, admin_user, organization):
"""
Constructed inventory hosts are still visible through the API
"""
source_inv = Inventory.objects.create(name='source-inv', organization=organization)
source_host = source_inv.hosts.create(name='host1')
constructed = Inventory.objects.create(name='constructed-inv', kind='constructed', organization=organization)
Host.objects.create(name='host1', inventory=constructed, instance_id=str(source_host.pk))
response = get(reverse('api:host_list'), user=admin_user, expect=200)
assert response.data['count'] == 2

View File

@@ -7,7 +7,7 @@ from django.core.exceptions import ValidationError
from awx.api.versioning import reverse
from awx.main.models import InventorySource, Inventory, ActivityStream, Organization
from awx.main.models import InventorySource, Inventory, ActivityStream
from awx.main.utils.inventory_vars import update_group_variables
@@ -463,26 +463,6 @@ class TestInventorySourceCredential:
assert 'Cloud-based inventory sources (such as ec2)' in r.data['credential'][0]
assert 'require credentials for the matching cloud service' in r.data['credential'][0]
def test_credential_dict_value_returns_400(self, inventory, admin_user, put):
"""Passing a dict for the credential field should return 400, not 500.
Reproduces a bug where int() raises TypeError on non-scalar types
(dict, list) which was uncaught, resulting in a 500 Internal Server Error.
"""
inv_src = InventorySource.objects.create(name='test-src', inventory=inventory, source='ec2')
r = put(
url=reverse('api:inventory_source_detail', kwargs={'pk': inv_src.pk}),
data={
'name': 'test-src',
'inventory': inventory.pk,
'source': 'ec2',
'credential': {'username': 'admin', 'password': 'secret'},
},
user=admin_user,
expect=400,
)
assert r.status_code == 400
def test_vault_credential_not_allowed(self, project, inventory, vault_credential, admin_user, post):
"""Vault credentials cannot be associated via the deprecated field"""
# TODO: when feature is added, add tests to use the related credentials
@@ -963,45 +943,3 @@ class TestInventoryAllVariables:
# Test step 6: Value of var x from source A reappears, because the
# latest update from source B did not contain var x.
self.update_and_verify(inv_src_c, {}, expect={"x": 1}, teststep=6)
@pytest.mark.django_db
def test_inventory_names_unique_per_organization(post, admin_user):
"""Validate that two inventories can have the same name if they belong to different organizations."""
org1 = Organization.objects.create(name='org-inv-1')
org2 = Organization.objects.create(name='org-inv-2')
inv_name = 'SharedInventoryName'
# Create inventory with same name in org1
resp1 = post(
reverse('api:inventory_list'),
{'name': inv_name, 'organization': org1.id},
admin_user,
expect=201,
)
inv1_id = resp1.data['id']
# Create inventory with same name in org2 - should succeed
resp2 = post(
reverse('api:inventory_list'),
{'name': inv_name, 'organization': org2.id},
admin_user,
expect=201,
)
inv2_id = resp2.data['id']
assert inv1_id != inv2_id
inv1 = Inventory.objects.get(id=inv1_id)
inv2 = Inventory.objects.get(id=inv2_id)
assert inv1.name == inv2.name == inv_name
assert inv1.organization.id == org1.id
assert inv2.organization.id == org2.id
# Attempt to create another inventory with same name in org1 - should fail
resp3 = post(
reverse('api:inventory_list'),
{'name': inv_name, 'organization': org1.id},
admin_user,
expect=400,
)
assert 'Inventory with this Name and Organization already exists' in json.dumps(resp3.data)

View File

@@ -1,92 +0,0 @@
# -*- coding: utf-8 -*-
import pytest
from awx.api.versioning import reverse
from awx.main.models import NotificationTemplate, Organization
@pytest.mark.django_db
def test_notification_template_names_unique_per_organization(post, admin_user):
"""
Validate that notification templates must have unique names within an organization,
but can have the same name across different organizations.
"""
org1 = Organization.objects.create(name='org-notif-1')
org2 = Organization.objects.create(name='org-notif-2')
template_name = 'SharedNotificationName'
# Create notification template in org1
resp1 = post(
reverse('api:notification_template_list'),
{
'name': template_name,
'organization': org1.id,
'notification_type': 'email',
'notification_configuration': {
'username': 'user@example.com',
'password': 'pass',
'sender': 'sender@example.com',
'recipients': ['recipient@example.com'],
'host': 'smtp.example.com',
'port': 25,
'use_tls': False,
'use_ssl': False,
},
},
admin_user,
expect=201,
)
template1_id = resp1.data['id']
# Create notification template with same name in org2 - should succeed
resp2 = post(
reverse('api:notification_template_list'),
{
'name': template_name,
'organization': org2.id,
'notification_type': 'email',
'notification_configuration': {
'username': 'user@example.com',
'password': 'pass',
'sender': 'sender@example.com',
'recipients': ['recipient@example.com'],
'host': 'smtp.example.com',
'port': 25,
'use_tls': False,
'use_ssl': False,
},
},
admin_user,
expect=201,
)
template2_id = resp2.data['id']
assert template1_id != template2_id
template1 = NotificationTemplate.objects.get(id=template1_id)
template2 = NotificationTemplate.objects.get(id=template2_id)
assert template1.name == template2.name == template_name
assert template1.organization.id == org1.id
assert template2.organization.id == org2.id
# Attempt to create another notification template with same name in org1 - should fail
resp3 = post(
reverse('api:notification_template_list'),
{
'name': template_name,
'organization': org1.id,
'notification_type': 'email',
'notification_configuration': {
'username': 'user@example.com',
'password': 'pass',
'sender': 'sender@example.com',
'recipients': ['recipient@example.com'],
'host': 'smtp.example.com',
'port': 25,
'use_tls': False,
'use_ssl': False,
},
},
admin_user,
expect=400,
)
assert 'Notification template with this Organization and Name already exists' in str(resp3.data)

View File

@@ -1,163 +0,0 @@
"""
Tests for OIDC workload identity credential type feature flag.
The FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED flag is an install-time flag that
controls whether OIDC credential types are loaded into the registry at startup.
When disabled, OIDC credential types are not loaded and do not exist in the database.
"""
import pytest
from unittest import mock
from django.test import override_settings
from awx.main.constants import OIDC_CREDENTIAL_TYPE_NAMESPACES
from awx.main.models.credential import CredentialType, ManagedCredentialType, load_credentials
from awx.api.versioning import reverse
@pytest.fixture
def reload_credentials_with_flag(django_db_setup, django_db_blocker):
"""
Fixture that reloads credentials with a specific flag state.
This simulates what happens at application startup.
"""
# Save original registry state
original_registry = ManagedCredentialType.registry.copy()
def _reload(flag_enabled):
with django_db_blocker.unblock():
# Clear the entire registry before reloading
ManagedCredentialType.registry.clear()
# Reload credentials with the specified flag state
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=flag_enabled):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
load_credentials()
# Sync to database
CredentialType.setup_tower_managed_defaults(lock=False)
# In tests, the session fixture pre-loads all credential types into the DB.
# Remove OIDC types when testing the disabled state so the API test is accurate.
if not flag_enabled:
CredentialType.objects.filter(namespace__in=OIDC_CREDENTIAL_TYPE_NAMESPACES).delete()
yield _reload
# Restore original registry state after tests
ManagedCredentialType.registry.clear()
ManagedCredentialType.registry.update(original_registry)
@pytest.fixture
def isolated_registry():
"""Save and restore the ManagedCredentialType registry, with full isolation via mocked entry_points."""
original_registry = ManagedCredentialType.registry.copy()
ManagedCredentialType.registry.clear()
yield
ManagedCredentialType.registry.clear()
ManagedCredentialType.registry.update(original_registry)
def _make_mock_entry_point(name):
"""Create a mock entry point that mimics a credential plugin."""
ep = mock.MagicMock()
ep.name = name
ep.value = f'test_plugin:{name}'
plugin = mock.MagicMock(spec=[])
ep.load.return_value = plugin
return ep
def _mock_entry_points_factory(managed_names, supported_names):
"""Return a side_effect function for mocking entry_points() with controlled plugins."""
managed = [_make_mock_entry_point(n) for n in managed_names]
supported = [_make_mock_entry_point(n) for n in supported_names]
def _entry_points(group):
if group == 'awx_plugins.managed_credentials':
return managed
elif group == 'awx_plugins.managed_credentials.supported':
return supported
return []
return _entry_points
# --- Unit tests for load_credentials() registry behavior ---
def test_oidc_types_in_registry_when_flag_enabled(isolated_registry):
"""Test that OIDC credential types are added to the registry when flag is enabled."""
mock_eps = _mock_entry_points_factory(
managed_names=['ssh', 'vault'],
supported_names=['hashivault-kv-oidc', 'hashivault-ssh-oidc'],
)
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=True):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
with mock.patch('awx.main.models.credential.entry_points', side_effect=mock_eps):
load_credentials()
for ns in OIDC_CREDENTIAL_TYPE_NAMESPACES:
assert ns in ManagedCredentialType.registry, f"{ns} should be in registry when flag is enabled"
assert 'ssh' in ManagedCredentialType.registry
assert 'vault' in ManagedCredentialType.registry
def test_oidc_types_not_in_registry_when_flag_disabled(isolated_registry):
"""Test that OIDC credential types are excluded from the registry when flag is disabled."""
mock_eps = _mock_entry_points_factory(
managed_names=['ssh', 'vault'],
supported_names=['hashivault-kv-oidc', 'hashivault-ssh-oidc'],
)
with override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=False):
with mock.patch('awx.main.models.credential.detect_server_product_name', return_value='NOT_AWX'):
with mock.patch('awx.main.models.credential.entry_points', side_effect=mock_eps):
load_credentials()
for ns in OIDC_CREDENTIAL_TYPE_NAMESPACES:
assert ns not in ManagedCredentialType.registry, f"{ns} should not be in registry when flag is disabled"
# Non-OIDC types should still be loaded
assert 'ssh' in ManagedCredentialType.registry
assert 'vault' in ManagedCredentialType.registry
def test_oidc_namespaces_constant():
"""Test that OIDC_CREDENTIAL_TYPE_NAMESPACES contains the expected namespaces."""
assert 'hashivault-kv-oidc' in OIDC_CREDENTIAL_TYPE_NAMESPACES
assert 'hashivault-ssh-oidc' in OIDC_CREDENTIAL_TYPE_NAMESPACES
assert len(OIDC_CREDENTIAL_TYPE_NAMESPACES) == 2
# --- Functional API tests ---
@pytest.mark.django_db
def test_oidc_types_loaded_when_flag_enabled(get, admin, reload_credentials_with_flag):
"""Test that OIDC credential types are visible in the API when flag is enabled."""
reload_credentials_with_flag(flag_enabled=True)
response = get(reverse('api:credential_type_list'), admin)
assert response.status_code == 200
namespaces = [ct['namespace'] for ct in response.data['results']]
assert 'hashivault-kv-oidc' in namespaces
assert 'hashivault-ssh-oidc' in namespaces
@pytest.mark.django_db
def test_oidc_types_not_loaded_when_flag_disabled(get, admin, reload_credentials_with_flag):
"""Test that OIDC credential types are not visible in the API when flag is disabled."""
reload_credentials_with_flag(flag_enabled=False)
response = get(reverse('api:credential_type_list'), admin)
assert response.status_code == 200
namespaces = [ct['namespace'] for ct in response.data['results']]
assert 'hashivault-kv-oidc' not in namespaces
assert 'hashivault-ssh-oidc' not in namespaces
# Verify they're also not in the database
assert not CredentialType.objects.filter(namespace='hashivault-kv-oidc').exists()
assert not CredentialType.objects.filter(namespace='hashivault-ssh-oidc').exists()

View File

@@ -1,311 +0,0 @@
"""
Tests for OIDC workload identity credential test endpoints.
Tests the /api/v2/credentials/<id>/test/ and /api/v2/credential_types/<id>/test/
endpoints when used with OIDC-enabled credential types.
"""
import pytest
from unittest import mock
from django.test import override_settings
from awx.main.models import Credential, CredentialType, JobTemplate
from awx.api.versioning import reverse
@pytest.fixture
def job_template(organization, project):
"""Job template with organization and project for OIDC JWT generation."""
return JobTemplate.objects.create(name='test-jt', organization=organization, project=project, playbook='helloworld.yml')
@pytest.fixture
def oidc_credentialtype():
"""Create a credential type with workload_identity_token internal field."""
oidc_type_inputs = {
'fields': [
{'id': 'url', 'label': 'Vault URL', 'type': 'string', 'help_text': 'The Vault server URL.'},
{'id': 'auth_path', 'label': 'Auth Path', 'type': 'string', 'help_text': 'JWT auth mount path.'},
{'id': 'role_id', 'label': 'Role ID', 'type': 'string', 'help_text': 'Vault role.'},
{'id': 'workload_identity_token', 'label': 'Workload Identity Token', 'type': 'string', 'secret': True, 'internal': True},
],
'metadata': [
{'id': 'secret_path', 'label': 'Secret Path', 'type': 'string'},
{'id': 'job_template_id', 'label': 'Job Template ID', 'type': 'string'},
],
'required': ['url', 'auth_path', 'role_id'],
}
class MockPlugin(object):
def backend(self, **kwargs):
# Simulate successful backend call
return 'secret'
with mock.patch('awx.main.models.credential.CredentialType.plugin', new_callable=mock.PropertyMock) as mock_plugin:
mock_plugin.return_value = MockPlugin()
oidc_type = CredentialType(kind='external', managed=True, namespace='hashivault-kv-oidc', name='HashiCorp Vault KV (OIDC)', inputs=oidc_type_inputs)
oidc_type.save()
yield oidc_type
@pytest.fixture
def oidc_credential(oidc_credentialtype):
"""Create a credential using the OIDC credential type."""
return Credential.objects.create(
credential_type=oidc_credentialtype,
name='oidc-vault-cred',
inputs={'url': 'http://vault.example.com:8200', 'auth_path': 'jwt', 'role_id': 'test-role'},
)
@pytest.fixture
def mock_oidc_backend():
"""Fixture that mocks OIDC JWT generation and credential backend."""
with mock.patch('awx.api.views.retrieve_workload_identity_jwt_with_claims') as mock_jwt, mock.patch('awx.api.views._jwt_decode') as mock_decode, mock.patch(
'awx.main.models.credential.CredentialType.plugin', new_callable=mock.PropertyMock
) as mock_plugin:
# Set default return values
mock_jwt.return_value = 'fake.jwt.token'
mock_decode.return_value = {'iss': 'http://gateway/o', 'aud': 'vault'}
# Create mock backend
mock_backend = mock.MagicMock()
mock_backend.backend.return_value = 'secret'
mock_plugin.return_value = mock_backend
# Yield all mocks for test customization
yield {
'jwt': mock_jwt,
'decode': mock_decode,
'plugin': mock_plugin,
'backend': mock_backend,
}
# --- Tests for CredentialExternalTest endpoint ---
@pytest.mark.django_db
@override_settings(FEATURE_OIDC_WORKLOAD_IDENTITY_ENABLED=False)
def test_credential_test_without_oidc_feature_flag(post, admin, oidc_credential):
"""Test that credential test works without OIDC feature flag enabled."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': '1'}}
with mock.patch('awx.main.models.credential.CredentialType.plugin', new_callable=mock.PropertyMock) as mock_plugin:
mock_backend = mock.MagicMock()
mock_backend.backend.return_value = 'secret'
mock_plugin.return_value = mock_backend
response = post(url, data, admin)
assert response.status_code == 202
# Should not contain JWT payload when feature flag is disabled
assert 'details' not in response.data or 'sent_jwt_payload' not in response.data.get('details', {})
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
@pytest.mark.parametrize(
'job_template_id, expected_error',
[
(None, 'Job template ID is required'),
('not-an-integer', 'must be an integer'),
('99999', 'does not exist'),
],
ids=['missing_job_template_id', 'invalid_job_template_id_type', 'nonexistent_job_template_id'],
)
def test_credential_test_job_template_validation(mock_flag, post, admin, oidc_credential, job_template_id, expected_error):
"""Test that invalid job_template_id values return 400 with appropriate error messages."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret'}}
if job_template_id is not None:
data['metadata']['job_template_id'] = job_template_id
response = post(url, data, admin)
assert response.status_code == 400
assert 'details' in response.data
assert 'error_message' in response.data['details']
assert expected_error in response.data['details']['error_message']
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_no_access_to_job_template(mock_flag, post, alice, oidc_credential, job_template):
"""Test that user without access to job template gets 403."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
# Give alice use permission on credential but not on job template
oidc_credential.use_role.members.add(alice)
response = post(url, data, alice)
assert response.status_code == 403
assert 'You do not have access to job template' in str(response.data)
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_success_returns_jwt_payload(mock_flag, post, admin, oidc_credential, job_template, mock_oidc_backend):
"""Test that successful test returns JWT payload in response."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
# Customize mock for this test
mock_oidc_backend['decode'].return_value = {
'iss': 'http://gateway/o',
'sub': 'system:serviceaccount:default:awx-operator',
'aud': 'vault',
'job_template_id': job_template.id,
}
response = post(url, data, admin)
assert response.status_code == 202
assert 'details' in response.data
assert 'sent_jwt_payload' in response.data['details']
assert response.data['details']['sent_jwt_payload']['job_template_id'] == job_template.id
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_response_does_not_contain_secret_value(mock_flag, post, admin, oidc_credential, job_template, mock_oidc_backend):
"""
the OIDC credential test endpoint must not echo the resolved Vault secret back to the caller.
"""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
credential_secret_value = 'CREDENTIAL_SECRET'
mock_oidc_backend['backend'].backend.return_value = credential_secret_value
response = post(url, data, admin)
assert response.status_code == 202
assert 'details' in response.data
assert 'sent_jwt_payload' in response.data['details']
assert 'secret_value' not in response.data['details']
assert credential_secret_value not in str(response.data)
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_backend_failure_returns_jwt_and_error(mock_flag, post, admin, oidc_credential, job_template, mock_oidc_backend):
"""Test that backend failure still returns JWT payload along with error message."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
# Make backend fail
mock_oidc_backend['backend'].backend.side_effect = RuntimeError('Connection failed')
response = post(url, data, admin)
assert response.status_code == 400
assert 'details' in response.data
# Both JWT payload and error message should be present
assert 'sent_jwt_payload' in response.data['details']
assert 'error_message' in response.data['details']
assert 'Connection failed' in response.data['details']['error_message']
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_jwt_generation_failure(mock_flag, post, admin, oidc_credential, job_template):
"""Test that JWT generation failure returns error without JWT payload."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
with mock.patch('awx.api.views.OIDCCredentialTestMixin._get_workload_identity_token') as mock_jwt:
mock_jwt.side_effect = RuntimeError('Failed to generate JWT')
response = post(url, data, admin)
assert response.status_code == 400
assert 'details' in response.data
assert 'error_message' in response.data['details']
assert 'Failed to generate JWT' in response.data['details']['error_message']
# No JWT payload when generation fails
assert 'sent_jwt_payload' not in response.data['details']
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_test_job_template_id_not_passed_to_backend(mock_flag, post, admin, oidc_credential, job_template, mock_oidc_backend):
"""Test that job_template_id is removed from backend_kwargs."""
url = reverse('api:credential_external_test', kwargs={'pk': oidc_credential.pk})
data = {'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)}}
response = post(url, data, admin)
assert response.status_code == 202
# Check that backend was called without job_template_id but with url and workload_identity_token
call_kwargs = mock_oidc_backend['backend'].backend.call_args[1]
assert 'job_template_id' not in call_kwargs
assert 'url' in call_kwargs
assert 'workload_identity_token' in call_kwargs
# --- Tests for CredentialTypeExternalTest endpoint ---
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_type_test_response_does_not_contain_secret_value(mock_flag, post, admin, oidc_credentialtype, job_template, mock_oidc_backend):
"""
the credential-type variant of the test endpoint should not return the secret value
"""
url = reverse('api:credential_type_external_test', kwargs={'pk': oidc_credentialtype.pk})
data = {
'inputs': {'url': 'http://vault.example.com:8200', 'auth_path': 'jwt', 'role_id': 'test-role', 'jwt_aud': 'vault'},
'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)},
}
credential_type_seret_value = 'CREDENTIAL_TYPE_SECRET'
mock_oidc_backend['backend'].backend.return_value = credential_type_seret_value
response = post(url, data, admin)
assert response.status_code == 202
assert 'details' in response.data
assert 'sent_jwt_payload' in response.data['details']
assert 'secret_value' not in response.data['details']
assert credential_type_seret_value not in str(response.data)
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_type_test_missing_job_template_id(mock_flag, post, admin, oidc_credentialtype):
"""Test that missing job_template_id returns 400 for credential type test endpoint."""
url = reverse('api:credential_type_external_test', kwargs={'pk': oidc_credentialtype.pk})
data = {
'inputs': {'url': 'http://vault.example.com:8200', 'auth_path': 'jwt', 'role_id': 'test-role'},
'metadata': {'secret_path': 'test/secret'},
}
response = post(url, data, admin)
assert response.status_code == 400
assert 'details' in response.data
assert 'error_message' in response.data['details']
assert 'Job template ID is required' in response.data['details']['error_message']
@pytest.mark.django_db
@mock.patch('awx.api.views.flag_enabled', return_value=True)
def test_credential_type_test_success_returns_jwt_payload(mock_flag, post, admin, oidc_credentialtype, job_template, mock_oidc_backend):
"""Test that successful credential type test returns JWT payload."""
url = reverse('api:credential_type_external_test', kwargs={'pk': oidc_credentialtype.pk})
data = {
'inputs': {'url': 'http://vault.example.com:8200', 'auth_path': 'jwt', 'role_id': 'test-role'},
'metadata': {'secret_path': 'test/secret', 'job_template_id': str(job_template.id)},
}
response = post(url, data, admin)
assert response.status_code == 202
assert 'details' in response.data
assert 'sent_jwt_payload' in response.data['details']
@pytest.mark.django_db
def test_credential_external_test_returns_400_for_non_external_credential(post, admin, credential):
# credential fixture creates a non-external credential (e.g. SSH/vault kind)
url = reverse('api:credential_external_test', kwargs={'pk': credential.pk})
response = post(url, {'metadata': {}}, admin)
assert response.status_code == 400
assert 'not testable' in response.data.get('detail', '').lower()

Some files were not shown because too many files have changed in this diff Show More