🧪 Gather coverage @ CI and upload to Codecov (#15499)

.codecov.yml

@@ -0,0 +1,49 @@
+---
+
+codecov:
+  notify:
+    after_n_builds: 1  # Number of test matrix+lint jobs uploading coverage
+    wait_for_ci: false
+
+  require_ci_to_pass: false
+
+  token: >-  # repo-scoped, upload-only, needed for stability in PRs from forks
+    2b8c7a7a-7293-4a00-bf02-19bd55a1389b
+
+comment:
+  require_changes: true
+
+coverage:
+  range: 100..100
+  status:
+    patch:
+      default:
+        target: 100%
+      pytest:
+        target: 100%
+        flags:
+          - pytest
+      typing:
+        flags:
+          - MyPy
+    project:
+      default:
+        target: 100%
+      lib:
+        flags:
+          - pytest
+        paths:
+          - src/
+        target: 100%
+      tests:
+        flags:
+          - pytest
+        paths:
+          - tests/
+        target: 100%
+      typing:
+        flags:
+          - MyPy
+        target: 100%
+
+...

.github/workflows/ci.yml

@@ -5,6 +5,7 @@ env:
   CI_GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
   DEV_DOCKER_OWNER: ${{ github.repository_owner }}
   COMPOSE_TAG: ${{ github.base_ref || 'devel' }}
+  UPSTREAM_REPOSITORY_ID: 91594105
 on:
   pull_request:
 jobs:
@@ -20,7 +21,7 @@ jobs:
       matrix:
         tests:
           - name: api-test
-            command: /start_tests.sh
+            command: /start_tests.sh test_coverage
           - name: api-migrations
             command: /start_tests.sh test_migrations
           - name: api-lint
@@ -43,7 +44,48 @@ jobs:
           github-token: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Run check ${{ matrix.tests.name }}
-        run: AWX_DOCKER_CMD='${{ matrix.tests.command }}' make docker-runner
+        id: make-run
+        run: >-
+          AWX_DOCKER_ARGS='-e GITHUB_ACTIONS -e GITHUB_OUTPUT -v "${GITHUB_OUTPUT}:${GITHUB_OUTPUT}:rw,Z"'
+          AWX_DOCKER_CMD='${{ matrix.tests.command }}'
+          make docker-runner
+
+      - name: Upload test coverage to Codecov
+        if: >-
+          !cancelled()
+          && matrix.tests.name == 'api-test'
+          && steps.make-run.outputs.cov-report-files != ''
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+      - name: Upload test results to Codecov
+        if: >-
+          !cancelled()
+          && matrix.tests.name == 'api-test'
+          && steps.make-run.outputs.test-result-files != ''
+        uses: codecov/test-results-action@v1
+        with:
+          fail_ci_if_error: >-
+            ${{
+              toJSON(env.UPSTREAM_REPOSITORY_ID == github.repository_id)
+            }}
+          flags: >-
+            CI-GHA,
+            pytest,
+            OS-${{
+              runner.os
+            }}
+          token: ${{ secrets.CODECOV_TOKEN }}
 
   dev-env:
     runs-on: ubuntu-latest

Makefile

@@ -351,6 +351,15 @@ test:
 	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
 	awx-manage check_migrations --dry-run --check  -n 'missing_migration_file'
 
+## Run all API unit tests with coverage enabled.
+test_coverage:
+	$(MAKE) test PYTEST_ARGS="--create-db --cov=awx --cov-report=xml --junitxml=reports/junit.xml"
+	@if [ "${GITHUB_ACTIONS}" = "true" ]; \
+	then \
+	  echo 'cov-report-files=coverage.xml' >> "${GITHUB_OUTPUT}"; \
+	  echo 'test-result-files=reports/junit.xml' >> "${GITHUB_OUTPUT}"; \
+	fi
+
 test_migrations:
 	if [ "$(VENV_BASE)" ]; then \
 		. $(VENV_BASE)/awx/bin/activate; \
@@ -359,7 +368,7 @@ test_migrations:
 
 ## Runs AWX_DOCKER_CMD inside a new docker container.
 docker-runner:
-	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
+	docker run -u $(shell id -u) --rm -v $(shell pwd):/awx_devel/:Z $(AWX_DOCKER_ARGS) --workdir=/awx_devel $(DEVEL_IMAGE_NAME) $(AWX_DOCKER_CMD)
 
 test_collection:
 	rm -f $(shell ls -d $(VENV_BASE)/awx/lib/python* | head -n 1)/no-global-site-packages.txt
@@ -414,13 +423,6 @@ test_unit:
 	fi; \
 	py.test awx/main/tests/unit awx/conf/tests/unit awx/sso/tests/unit
 
-## Run all API unit tests with coverage enabled.
-test_coverage:
-	@if [ "$(VENV_BASE)" ]; then \
-		. $(VENV_BASE)/awx/bin/activate; \
-	fi; \
-	py.test --create-db --cov=awx --cov-report=xml --junitxml=./reports/junit.xml $(TEST_DIRS)
-
 ## Output test coverage as HTML (into htmlcov directory).
 coverage_html:
 	coverage html