From 1a35775c25f688fbdf3fd9b6914605e119185aba Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Fri, 6 Dec 2024 09:20:26 -0500 Subject: [PATCH] Create a new pytest folder for live system testing with normal services (#15688) * PoC for running dev env tests * Replace in github actions * Try non interactive * Move folder to better location * Further streamlining of new test folders * Consolidate fixture, add writeup docs * Use star import * Push the wait-for-job to the conftest --- .github/workflows/ci.yml | 4 +- Makefile | 3 ++ awx/main/tests/README.md | 42 +++++++++++++++ awx/main/tests/live/pytest.ini | 3 ++ awx/main/tests/live/pytest_django_config.py | 12 +++++ awx/main/tests/live/tests/conftest.py | 28 ++++++++++ awx/main/tests/live/tests/test_demo_data.py | 15 ++++++ tools/docker-compose/ansible/smoke-test.yml | 60 --------------------- 8 files changed, 105 insertions(+), 62 deletions(-) create mode 100644 awx/main/tests/README.md create mode 100644 awx/main/tests/live/pytest.ini create mode 100644 awx/main/tests/live/pytest_django_config.py create mode 100644 awx/main/tests/live/tests/conftest.py create mode 100644 awx/main/tests/live/tests/test_demo_data.py delete mode 100644 tools/docker-compose/ansible/smoke-test.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 08479c3946..4b2ec43e6a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -139,8 +139,8 @@ jobs: build-ui: false github-token: ${{ secrets.GITHUB_TOKEN }} - - name: Run smoke test - run: ansible-playbook tools/docker-compose/ansible/smoke-test.yml -v + - name: Run live dev env tests + run: docker exec tools_awx_1 /bin/bash -c "make live_test" awx-operator: runs-on: ubuntu-latest diff --git a/Makefile b/Makefile index e9a1495644..3bf198f417 100644 --- a/Makefile +++ b/Makefile @@ -350,6 +350,9 @@ test: cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3 awx-manage check_migrations --dry-run --check -n 'missing_migration_file' +live_test: + cd awx/main/tests/live && py.test tests/ + ## Run all API unit tests with coverage enabled. test_coverage: $(MAKE) test PYTEST_ARGS="--create-db --cov --cov-report=xml --junitxml=reports/junit.xml" diff --git a/awx/main/tests/README.md b/awx/main/tests/README.md new file mode 100644 index 0000000000..f6aac6418e --- /dev/null +++ b/awx/main/tests/README.md @@ -0,0 +1,42 @@ +## Test Environments + +Several of the subfolders of `awx/main/tests/` indicate a different required _environment_ +where you can run the tests. Those folders are: + + - `functional/` - requires a test database and no other services running + - `live/` - must run in `tools_awx_1` container launched by `make docker-compose` + - `unit/` - does not require a test database or any active services + +### Functional and unit test environment + +The functional and unit tests have an invocation in `make test`, +and this attaches several other things like schema that piggybacks on requests. +These tests are ran from the root AWX folder. + +#### Functional tests + +Only tests in the `functional/` folder should use the `@pytest.mark.django_db` decorator. +This is the only difference between the functional and unit folders, +the test environment is otherwise the same for both. + +Functional tests use a sqlite3 database, so the postgres service is not necessary. + +### Live tests + +The live tests have an invocation in `make live_test` which will change +directory before running, which is required to pick up a different pytest +configuration. + +This will use the postges container from `make docker-compose` for the database, +and will disable the pytest-django features of running with a test database +and running tests in transactions. +This means that any changes done in the course of the test could potentially +be seen in your browser via the API or UI, and anything the test fails +to clean up will remain in the database. + +### Folders that should not contain tests + + - `data/` - just files other tests use + - `docs/` - utilities for schema generation + - `factories/` - general utilities + - `manual/` - python files to be ran directly diff --git a/awx/main/tests/live/pytest.ini b/awx/main/tests/live/pytest.ini new file mode 100644 index 0000000000..cc1e9c5a91 --- /dev/null +++ b/awx/main/tests/live/pytest.ini @@ -0,0 +1,3 @@ +# This file is needed to undo the pytest settings from the project root +[pytest] +addopts = -p no:django -p awx.main.tests.live.pytest_django_config diff --git a/awx/main/tests/live/pytest_django_config.py b/awx/main/tests/live/pytest_django_config.py new file mode 100644 index 0000000000..8c4eac2474 --- /dev/null +++ b/awx/main/tests/live/pytest_django_config.py @@ -0,0 +1,12 @@ +import django + +from awx import prepare_env + + +def pytest_load_initial_conftests(args): + """Replacement for same-named method in pytest_django plugin + + Instead of setting up a test database, this just sets up Django normally + this will give access to the postgres database as-is, for better and worse""" + prepare_env() + django.setup() diff --git a/awx/main/tests/live/tests/conftest.py b/awx/main/tests/live/tests/conftest.py new file mode 100644 index 0000000000..848e8fd7d2 --- /dev/null +++ b/awx/main/tests/live/tests/conftest.py @@ -0,0 +1,28 @@ +import time + +# These tests are invoked from the awx/main/tests/live/ subfolder +# so any fixtures from higher-up conftest files must be explicitly included +from awx.main.tests.functional.conftest import * # noqa + + +def wait_to_leave_status(job, status, timeout=25, sleep_time=0.1): + """Wait until the job does NOT have the specified status with some timeout + + the default timeout of 25 if chosen because the task manager runs on a 20 second + schedule, and the API does not guarentee working jobs faster than this + """ + start = time.time() + while time.time() - start < timeout: + job.refresh_from_db() + if job.status != status: + return + time.sleep(sleep_time) + raise RuntimeError(f'Job failed to exit {status} in {timeout} seconds. job_explanation={job.job_explanation} tb={job.result_traceback}') + + +def wait_for_job(job, final_status='successful', running_timeout=800): + wait_to_leave_status(job, 'pending') + wait_to_leave_status(job, 'waiting') + wait_to_leave_status(job, 'running', timeout=running_timeout) + + assert job.status == final_status, f'Job was not successful id={job.id} status={job.status}' diff --git a/awx/main/tests/live/tests/test_demo_data.py b/awx/main/tests/live/tests/test_demo_data.py new file mode 100644 index 0000000000..fa9ee5eb97 --- /dev/null +++ b/awx/main/tests/live/tests/test_demo_data.py @@ -0,0 +1,15 @@ +from awx.api.versioning import reverse + +from awx.main.models import JobTemplate, Job + +from awx.main.tests.live.tests.conftest import wait_for_job + + +def test_launch_demo_jt(post, admin): + jt = JobTemplate.objects.get(name='Demo Job Template') + + url = reverse('api:job_template_launch', kwargs={'pk': jt.id}) + + r = post(url=url, data={}, user=admin, expect=201) + job = Job.objects.get(pk=r.data['id']) + wait_for_job(job) diff --git a/tools/docker-compose/ansible/smoke-test.yml b/tools/docker-compose/ansible/smoke-test.yml deleted file mode 100644 index 591ceed396..0000000000 --- a/tools/docker-compose/ansible/smoke-test.yml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# -# This is used by a CI check in GitHub Actions and isnt really -# meant to be run locally. -# -# The development environment does some unfortunate things to -# make rootless podman work inside of a docker container. -# The goal here is to essentially tests that the awx user is -# able to run `podman run`. -# -- name: Test that the development environment is able to launch a job - hosts: localhost - tasks: - - name: Reset admin password - shell: | - docker exec -i tools_awx_1 bash <