AAP-84057: Set PostgreSQL statement_timeout on web worker DB connections (#16558)

* AAP-84057: Set PostgreSQL statement_timeout on web worker DB connections

When uwsgi's harakiri kills a worker, the PostgreSQL backend continues
running the query indefinitely.  These abandoned queries accumulate and
create resource contention for all other queries.

Add a connection_created signal handler that sets statement_timeout on
new DB connections.  Under uwsgi, the timeout is auto-derived from the
harakiri value (minus 5s margin so PostgreSQL cancels the query before
uwsgi kills the worker).  Outside uwsgi (task workers, migrations),
no timeout is applied.  A manual DATABASE_STATEMENT_TIMEOUT setting
is available as a fallback for non-uwsgi deployments.

* Remove timeout value caching because it brings no significant gains

* Use proportional margin between statement_timeout and harakiri timeout

* Fix zero-harakiri test to patch fake uwsgi module instead of None

* Refactor statement_timeout from signal handler to connection string

Move statement_timeout configuration from a connection_created signal
handler (extra SQL round-trip per connection) to a dynaconf merge
function that sets it via the libpq OPTIONS connection string parameter.
This mirrors the existing merge_application_name() pattern and
eliminates the SET statement on every new connection.

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

awx/main/tests/unit/test_statement_timeout.py

@@ -0,0 +1,79 @@
+import types
+from unittest import mock
+
+from awx.settings.functions import merge_statement_timeout
+
+PG_ENGINE = "django.db.backends.postgresql"
+SQLITE_ENGINE = "django.db.backends.sqlite3"
+
+
+def _make_settings(engine=PG_ENGINE, timeout=None, existing_options=""):
+    """Build a dict that quacks like DYNACONF.get() for merge_statement_timeout."""
+    data = {"DATABASES__default__ENGINE": engine}
+    if timeout is not None:
+        data["DATABASE_STATEMENT_TIMEOUT"] = timeout
+    if existing_options:
+        data["DATABASES__default__OPTIONS__options"] = existing_options
+    return data
+
+
+def _fake_uwsgi(harakiri):
+    mod = types.ModuleType('uwsgi')
+    mod.opt = {b'harakiri': str(harakiri).encode()}
+    return mod
+
+
+class TestMergeStatementTimeout:
+    def test_derives_from_uwsgi_harakiri(self):
+        settings = _make_settings()
+        with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(115)}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=110000"}
+
+    def test_returns_empty_without_uwsgi_or_setting(self):
+        settings = _make_settings()
+        with mock.patch.dict('sys.modules', {'uwsgi': None}):
+            result = merge_statement_timeout(settings)
+        assert result == {}
+
+    def test_falls_back_to_setting(self):
+        settings = _make_settings(timeout=60000)
+        with mock.patch.dict('sys.modules', {'uwsgi': None}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=60000"}
+
+    def test_uwsgi_takes_precedence_over_setting(self):
+        settings = _make_settings(timeout=60000)
+        with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(115)}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=110000"}
+
+    def test_harakiri_zero_falls_back_to_setting(self):
+        settings = _make_settings(timeout=90000)
+        with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(0)}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=90000"}
+
+    def test_harakiri_very_low_clamps_to_one_second(self):
+        settings = _make_settings()
+        with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(1)}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=1000"}
+
+    def test_harakiri_midrange_uses_proportional_margin(self):
+        settings = _make_settings()
+        with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(30)}):
+            # margin = min(5, max(1, int(30*0.1))) = 3 → timeout = 27s
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=27000"}
+
+    def test_skips_sqlite(self):
+        settings = _make_settings(engine=SQLITE_ENGINE, timeout=60000)
+        result = merge_statement_timeout(settings)
+        assert result == {}
+
+    def test_appends_to_existing_options(self):
+        settings = _make_settings(timeout=60000, existing_options="-c lock_timeout=5000")
+        with mock.patch.dict('sys.modules', {'uwsgi': None}):
+            result = merge_statement_timeout(settings)
+        assert result == {"DATABASES__default__OPTIONS__options": "-c lock_timeout=5000 -c statement_timeout=60000"}

awx/settings/__init__.py

@@ -12,6 +12,7 @@ from ansible_base.lib.dynamic_config import (
 from .functions import (
     assert_production_settings,
     merge_application_name,
+    merge_statement_timeout,
     add_backwards_compatibility,
     load_extra_development_files,
 )
@@ -78,6 +79,11 @@ DYNACONF.update(
     loader_identifier="awx.settings:merge_application_name",
     merge=True,
 )
+DYNACONF.update(
+    merge_statement_timeout(DYNACONF),
+    loader_identifier="awx.settings:merge_statement_timeout",
+    merge=True,
+)
 
 # Update django.conf.settings with DYNACONF values
 export(__name__, DYNACONF)

awx/settings/defaults.py

@@ -43,6 +43,11 @@ LISTENER_DATABASES = {
     }
 }
 
+# Optional manual override for statement_timeout (ms) on web worker DB
+# connections.  When running under uwsgi, the timeout is auto-derived from
+# the harakiri value.  Set this for non-uwsgi deployments or to override.
+DATABASE_STATEMENT_TIMEOUT = None
+
 # Whether or not the deployment is a K8S-based deployment
 # In K8S-based deployments, instances have zero capacity - all playbook
 # automation is intended to flow through defined Container Groups that

awx/settings/functions.py

@@ -12,6 +12,41 @@ def merge_application_name(settings):
     return data
 
 
+def merge_statement_timeout(settings):
+    """Return a dynaconf merge dict to set statement_timeout for web worker DB connections.
+
+    Under uwsgi, derives timeout from harakiri with a safety margin so
+    PostgreSQL cancels the query before uwsgi kills the worker.  The margin
+    is 10% of harakiri, clamped to [1s, 5s].  Falls back to the
+    DATABASE_STATEMENT_TIMEOUT setting for non-uwsgi deployments.
+    """
+    if "sqlite3" in settings.get("DATABASES__default__ENGINE", ""):
+        return {}
+
+    timeout_ms = None
+    try:
+        import uwsgi
+
+        harakiri = int(uwsgi.opt.get(b'harakiri', 0))
+        if harakiri > 0:
+            margin = min(5, max(1, int(harakiri * 0.1)))
+            timeout_ms = max(1000, (harakiri - margin) * 1000)
+    except (ImportError, ValueError):
+        pass
+
+    if timeout_ms is None:
+        timeout_ms = settings.get("DATABASE_STATEMENT_TIMEOUT")
+
+    if timeout_ms is None:
+        return {}
+
+    existing = settings.get("DATABASES__default__OPTIONS__options", "")
+    new_opt = f"-c statement_timeout={timeout_ms}"
+    value = f"{existing} {new_opt}".strip() if existing else new_opt
+
+    return {"DATABASES__default__OPTIONS__options": value}
+
+
 def add_backwards_compatibility():
     """Add backwards compatibility for AWX_MODE.