Create a new pytest folder for live system testing with normal services (#15688)

* PoC for running dev env tests

* Replace in github actions

* Try non interactive

* Move folder to better location

* Further streamlining of new test folders

* Consolidate fixture, add writeup docs

* Use star import

* Push the wait-for-job to the conftest

awx/main/tests/README.md

@@ -0,0 +1,42 @@
+## Test Environments
+
+Several of the subfolders of `awx/main/tests/` indicate a different required _environment_
+where you can run the tests. Those folders are:
+
+ - `functional/` - requires a test database and no other services running
+ - `live/` - must run in `tools_awx_1` container launched by `make docker-compose`
+ - `unit/` - does not require a test database or any active services
+
+### Functional and unit test environment
+
+The functional and unit tests have an invocation in `make test`,
+and this attaches several other things like schema that piggybacks on requests.
+These tests are ran from the root AWX folder.
+
+#### Functional tests
+
+Only tests in the `functional/` folder should use the `@pytest.mark.django_db` decorator.
+This is the only difference between the functional and unit folders,
+the test environment is otherwise the same for both.
+
+Functional tests use a sqlite3 database, so the postgres service is not necessary.
+
+### Live tests
+
+The live tests have an invocation in `make live_test` which will change
+directory before running, which is required to pick up a different pytest
+configuration.
+
+This will use the postges container from `make docker-compose` for the database,
+and will disable the pytest-django features of running with a test database
+and running tests in transactions.
+This means that any changes done in the course of the test could potentially
+be seen in your browser via the API or UI, and anything the test fails
+to clean up will remain in the database.
+
+### Folders that should not contain tests
+
+ - `data/` - just files other tests use
+ - `docs/` - utilities for schema generation
+ - `factories/` - general utilities
+ - `manual/` - python files to be ran directly

awx/main/tests/live/pytest.ini

@@ -0,0 +1,3 @@
+# This file is needed to undo the pytest settings from the project root
+[pytest]
+addopts =  -p no:django -p awx.main.tests.live.pytest_django_config

awx/main/tests/live/pytest_django_config.py

@@ -0,0 +1,12 @@
+import django
+
+from awx import prepare_env
+
+
+def pytest_load_initial_conftests(args):
+    """Replacement for same-named method in pytest_django plugin
+
+    Instead of setting up a test database, this just sets up Django normally
+    this will give access to the postgres database as-is, for better and worse"""
+    prepare_env()
+    django.setup()

awx/main/tests/live/tests/conftest.py

@@ -0,0 +1,28 @@
+import time
+
+# These tests are invoked from the awx/main/tests/live/ subfolder
+# so any fixtures from higher-up conftest files must be explicitly included
+from awx.main.tests.functional.conftest import *  # noqa
+
+
+def wait_to_leave_status(job, status, timeout=25, sleep_time=0.1):
+    """Wait until the job does NOT have the specified status with some timeout
+
+    the default timeout of 25 if chosen because the task manager runs on a 20 second
+    schedule, and the API does not guarentee working jobs faster than this
+    """
+    start = time.time()
+    while time.time() - start < timeout:
+        job.refresh_from_db()
+        if job.status != status:
+            return
+        time.sleep(sleep_time)
+    raise RuntimeError(f'Job failed to exit {status} in {timeout} seconds. job_explanation={job.job_explanation} tb={job.result_traceback}')
+
+
+def wait_for_job(job, final_status='successful', running_timeout=800):
+    wait_to_leave_status(job, 'pending')
+    wait_to_leave_status(job, 'waiting')
+    wait_to_leave_status(job, 'running', timeout=running_timeout)
+
+    assert job.status == final_status, f'Job was not successful id={job.id} status={job.status}'

awx/main/tests/live/tests/test_demo_data.py

@@ -0,0 +1,15 @@
+from awx.api.versioning import reverse
+
+from awx.main.models import JobTemplate, Job
+
+from awx.main.tests.live.tests.conftest import wait_for_job
+
+
+def test_launch_demo_jt(post, admin):
+    jt = JobTemplate.objects.get(name='Demo Job Template')
+
+    url = reverse('api:job_template_launch', kwargs={'pk': jt.id})
+
+    r = post(url=url, data={}, user=admin, expect=201)
+    job = Job.objects.get(pk=r.data['id'])
+    wait_for_job(job)