🧪 Make pytest notify us about future warnings

In essence, this configures Python to turn any warnings emitted in
runtime into errors[[1]]. This is the best practice that allows
reacting to future deprecation announcements that are coming from the
dependencies (direct, or transitive, or even CPython itself)[[2]].

The typical workflow looks like this:

  1. If a dependency is updated an a warning is hit in tests, the
     deprecated thing should be replaced with newer APIs.

  2. If a dependency is transitive or we have no control over it
     otherwise, the specific warning and a regex matching its message,
     plus the module reference (where possible) can be added to the
     list of temporary ignores in `pytest.ini`.

  3. The list of temporary ignores should be reevaluated periodically,
     including when dependency re-pinning in lockfile is happening.

[1]: https://docs.python.org/3/using/cmdline.html#cmdoption-W
[2]: https://pytest-with-eric.com/configuration/pytest-ignore-warnings/

awx_collection/test/awx/test_instance.py

@@ -8,6 +8,12 @@ from awx.main.models import Instance
 from django.test.utils import override_settings
 
 
+@pytest.mark.filterwarnings(
+    # FIXME: Figure out where it is emited and what causes it.
+    # FIXME: The suppression should be made more specific or the cause fixed.
+    # Ref: https://github.com/ansible/awx/pull/15620
+    "ignore::RuntimeWarning",
+)
 @pytest.mark.django_db
 def test_peers_adding_and_removing(run_module, admin_user):
     with override_settings(IS_K8S=True):

awx_collection/test/awx/test_instance_group.py

@@ -7,6 +7,12 @@ import pytest
 from awx.main.models import InstanceGroup, Instance
 
 
+@pytest.mark.filterwarnings(
+    # FIXME: Figure out where it is emited and what causes it.
+    # FIXME: The suppression should be made more specific or the cause fixed.
+    # Ref: https://github.com/ansible/awx/pull/15620
+    "ignore::RuntimeWarning",
+)
 @pytest.mark.django_db
 def test_instance_group_create(run_module, admin_user):
     result = run_module(
@@ -36,6 +42,12 @@ def test_instance_group_create(run_module, admin_user):
     assert len(all_instance_names) == 1, 'Too many instances in group {0}'.format(','.join(all_instance_names))
 
 
+@pytest.mark.filterwarnings(
+    # FIXME: Figure out where it is emited and what causes it.
+    # FIXME: The suppression should be made more specific or the cause fixed.
+    # Ref: https://github.com/ansible/awx/pull/15620
+    "ignore::RuntimeWarning",
+)
 @pytest.mark.django_db
 def test_container_group_create(run_module, admin_user, kube_credential):
     pod_spec = "{ 'Nothing': True }"

awx_collection/test/awx/test_workflow_job_template_node.py

@@ -8,6 +8,14 @@ import pytest
 from awx.main.models import WorkflowJobTemplateNode, WorkflowJobTemplate, JobTemplate, UnifiedJobTemplate
 
 
+pytestmark = pytest.mark.filterwarnings(
+    # FIXME: Figure out where it is emited and what causes it.
+    # FIXME: The suppression should be made more specific or the cause fixed.
+    # Ref: https://github.com/ansible/awx/pull/15620
+    "ignore::RuntimeWarning",
+)
+
+
 @pytest.fixture
 def job_template(project, inventory):
     return JobTemplate.objects.create(

awxkit/tox.ini

@@ -44,4 +44,8 @@ max-line-length = 120
 
 [pytest]
 addopts = -v --tb=native
+
+filterwarnings =
+  error
+
 junit_family=xunit2

pytest.ini

@@ -12,6 +12,93 @@ markers =
     job_runtime_vars:
     fixture_args:
 
+filterwarnings =
+  error
+
+  # FIXME: Set `USE_TZ` to `True`.
+  once:The default value of USE_TZ will change from False to True in Django 5.0. Set USE_TZ to False in your project settings if you want to keep the current default behavior.:django.utils.deprecation.RemovedInDjango50Warning:django.conf
+
+  # FIXME: Delete this entry once `USE_L10N` use is removed.
+  once:The USE_L10N setting is deprecated. Starting with Django 5.0, localized formatting of data will always be enabled. For example Django will display numbers and dates using the format of the current locale.:django.utils.deprecation.RemovedInDjango50Warning:django.conf
+
+  # FIXME: Delete this entry once `pyparsing` is updated.
+  once:module 'sre_constants' is deprecated:DeprecationWarning:_pytest.assertion.rewrite
+
+  # FIXME: Delete this entry once `polymorphic` is updated.
+  once:pkg_resources is deprecated as an API. See https.//setuptools.pypa.io/en/latest/pkg_resources.html:DeprecationWarning:_pytest.assertion.rewrite
+
+  # FIXME: Delete this entry once `zope` is updated.
+  once:Deprecated call to `pkg_resources.declare_namespace.'zope'.`.\nImplementing implicit namespace packages .as specified in PEP 420. is preferred to `pkg_resources.declare_namespace`. See https.//setuptools.pypa.io/en/latest/references/keywords.html#keyword-namespace-packages:DeprecationWarning:
+
+  # FIXME: Delete this entry once `coreapi` is updated.
+  once:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning:_pytest.assertion.rewrite
+
+  # FIXME: Delete this entry once the use of `distutils` is exterminated from the repo.
+  once:The distutils package is deprecated and slated for removal in Python 3.12. Use setuptools or check PEP 632 for potential alternatives:DeprecationWarning:_pytest.assertion.rewrite
+
+  # FIXME: Delete this entry once `coreapi` is deleted from the dependencies
+  # FIXME: and is no longer imported at runtime.
+  once:CoreAPI compatibility is deprecated and will be removed in DRF 3.17:rest_framework.RemovedInDRF317Warning:rest_framework.schemas.coreapi
+
+  # FIXME: Delete this entry once naive dates aren't passed to DB lookup
+  # FIXME: methods. Not sure where, might be in awx's views or in DAB.
+  once:DateTimeField User.date_joined received a naive datetime .2020-01-01 00.00.00. while time zone support is active.:RuntimeWarning:django.db.models.fields
+
+  # FIXME: Delete this entry once the deprecation is acted upon.
+  once:'index_together' is deprecated. Use 'Meta.indexes' in 'main.\w+' instead.:django.utils.deprecation.RemovedInDjango51Warning:django.db.models.options
+
+  # FIXME: Update `awx.main.migrations._dab_rbac` and delete this entry.
+  # once:Using QuerySet.iterator.. after prefetch_related.. without specifying chunk_size is deprecated.:django.utils.deprecation.RemovedInDjango50Warning:django.db.models.query
+  once:Using QuerySet.iterator.. after prefetch_related.. without specifying chunk_size is deprecated.:django.utils.deprecation.RemovedInDjango50Warning:awx.main.migrations._dab_rbac
+
+  # FIXME: Delete this entry once the **broken** always-true assertions in the
+  # FIXME: following tests are fixed:
+  # * `awx/main/tests/unit/utils/test_common.py::TestHostnameRegexValidator::test_good_call`
+  # * `awx/main/tests/unit/utils/test_common.py::TestHostnameRegexValidator::test_bad_call_with_inverse`
+  once:assertion is always true, perhaps remove parentheses\?:pytest.PytestAssertRewriteWarning:
+
+  # FIXME: Figure this out, fix and then delete the entry. It's not entirely
+  # FIXME: clear what emits it and where.
+  once:Pagination may yield inconsistent results with an unordered object_list. .class 'awx.main.models.workflow.WorkflowJobTemplateNode'. QuerySet.:django.core.paginator.UnorderedObjectListWarning:django.core.paginator
+
+  # FIXME: Figure this out, fix and then delete the entry.
+  once::django.core.paginator.UnorderedObjectListWarning:rest_framework.pagination
+
+  # FIXME: Replace use of `distro.linux_distribution()` via a context manager
+  # FIXME: in `awx/main/analytics/collectors.py` and then delete the entry.
+  once:distro.linux_distribution.. is deprecated. It should only be used as a compatibility shim with Python's platform.linux_distribution... Please use distro.id.., distro.version.. and distro.name.. instead.:DeprecationWarning:awx.main.analytics.collectors
+
+  # FIXME: Figure this out, fix and then delete the entry.
+  once:\nUsing ProtocolTypeRouter without an explicit "http" key is deprecated.\nGiven that you have not passed the "http" you likely should use Django's\nget_asgi_application...:DeprecationWarning:awx.main.routing
+
+  # FIXME: Figure this out, fix and then delete the entry.
+  once:Channel's inbuilt http protocol AsgiHandler is deprecated. Use Django's get_asgi_application.. instead.:DeprecationWarning:channels.routing
+
+  # FIXME: Use `codecs.open()` via a context manager
+  # FIXME: in `awx/main/utils/ansible.py` to close hanging file descriptors
+  # FIXME: and then delete the entry.
+  once:unclosed file <_io.BufferedReader name='[^']+'>:ResourceWarning:awx.main.utils.ansible
+
+  # FIXME: Use `open()` via a context manager
+  # FIXME: in `awx/main/tests/unit/test_tasks.py` to close hanging file
+  # FIXME: descriptors and then delete the entry.
+  once:unclosed file <_io.TextIOWrapper name='[^']+' mode='r' encoding='UTF-8'>:ResourceWarning:awx.main.tests.unit.test_tasks
+
+  # FIXME: Use `open()` via a context manager
+  # FIXME: in `awx/main/tests/unit/test_tasks.py` to close hanging file
+  # FIXME: descriptors and then delete the entry.
+  once:unclosed file <_io.BufferedReader name='[^']+'>:ResourceWarning:awx.main.tests.unit.test_tasks
+
+  # FIXME: Use `open()` via a context manager
+  # FIXME: in `awx/main/tests/unit/test_tasks.py` to close hanging file
+  # FIXME: descriptors and then delete the entry.
+  once:unclosed file <_io.TextIOWrapper name='[^']+' mode='r' encoding='UTF-8'>:ResourceWarning:_pytest.python
+
+  # FIXME: Use `open()` via a context manager
+  # FIXME: in `awx/main/utils/common.py` to close hanging file descriptors
+  # FIXME: and then delete the entry.
+  once:unclosed file <_io.BufferedWriter name='[^']+'>:ResourceWarning:awx.main.utils.common
+
 # https://docs.pytest.org/en/stable/usage.html#creating-junitxml-format-files
 junit_duration_report = call
 # xunit1 contains more metadata than xunit2 so it's better for CI UIs: