From 2a28b80ec6f1637b98a5ba5a448bf94694c43495 Mon Sep 17 00:00:00 2001 From: Dirk Julich Date: Fri, 31 Jul 2026 10:27:33 +0200 Subject: [PATCH] AAP-84057: Set PostgreSQL statement_timeout on web worker DB connections (#16558) * AAP-84057: Set PostgreSQL statement_timeout on web worker DB connections When uwsgi's harakiri kills a worker, the PostgreSQL backend continues running the query indefinitely. These abandoned queries accumulate and create resource contention for all other queries. Add a connection_created signal handler that sets statement_timeout on new DB connections. Under uwsgi, the timeout is auto-derived from the harakiri value (minus 5s margin so PostgreSQL cancels the query before uwsgi kills the worker). Outside uwsgi (task workers, migrations), no timeout is applied. A manual DATABASE_STATEMENT_TIMEOUT setting is available as a fallback for non-uwsgi deployments. * Remove timeout value caching because it brings no significant gains * Use proportional margin between statement_timeout and harakiri timeout * Fix zero-harakiri test to patch fake uwsgi module instead of None * Refactor statement_timeout from signal handler to connection string Move statement_timeout configuration from a connection_created signal handler (extra SQL round-trip per connection) to a dynaconf merge function that sets it via the libpq OPTIONS connection string parameter. This mirrors the existing merge_application_name() pattern and eliminates the SET statement on every new connection. --------- Co-authored-by: Claude Opus 4.6 --- awx/main/tests/unit/test_statement_timeout.py | 79 +++++++++++++++++++ awx/settings/__init__.py | 6 ++ awx/settings/defaults.py | 5 ++ awx/settings/functions.py | 35 ++++++++ 4 files changed, 125 insertions(+) create mode 100644 awx/main/tests/unit/test_statement_timeout.py diff --git a/awx/main/tests/unit/test_statement_timeout.py b/awx/main/tests/unit/test_statement_timeout.py new file mode 100644 index 0000000000..ba2ef133c2 --- /dev/null +++ b/awx/main/tests/unit/test_statement_timeout.py @@ -0,0 +1,79 @@ +import types +from unittest import mock + +from awx.settings.functions import merge_statement_timeout + +PG_ENGINE = "django.db.backends.postgresql" +SQLITE_ENGINE = "django.db.backends.sqlite3" + + +def _make_settings(engine=PG_ENGINE, timeout=None, existing_options=""): + """Build a dict that quacks like DYNACONF.get() for merge_statement_timeout.""" + data = {"DATABASES__default__ENGINE": engine} + if timeout is not None: + data["DATABASE_STATEMENT_TIMEOUT"] = timeout + if existing_options: + data["DATABASES__default__OPTIONS__options"] = existing_options + return data + + +def _fake_uwsgi(harakiri): + mod = types.ModuleType('uwsgi') + mod.opt = {b'harakiri': str(harakiri).encode()} + return mod + + +class TestMergeStatementTimeout: + def test_derives_from_uwsgi_harakiri(self): + settings = _make_settings() + with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(115)}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=110000"} + + def test_returns_empty_without_uwsgi_or_setting(self): + settings = _make_settings() + with mock.patch.dict('sys.modules', {'uwsgi': None}): + result = merge_statement_timeout(settings) + assert result == {} + + def test_falls_back_to_setting(self): + settings = _make_settings(timeout=60000) + with mock.patch.dict('sys.modules', {'uwsgi': None}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=60000"} + + def test_uwsgi_takes_precedence_over_setting(self): + settings = _make_settings(timeout=60000) + with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(115)}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=110000"} + + def test_harakiri_zero_falls_back_to_setting(self): + settings = _make_settings(timeout=90000) + with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(0)}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=90000"} + + def test_harakiri_very_low_clamps_to_one_second(self): + settings = _make_settings() + with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(1)}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=1000"} + + def test_harakiri_midrange_uses_proportional_margin(self): + settings = _make_settings() + with mock.patch.dict('sys.modules', {'uwsgi': _fake_uwsgi(30)}): + # margin = min(5, max(1, int(30*0.1))) = 3 → timeout = 27s + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c statement_timeout=27000"} + + def test_skips_sqlite(self): + settings = _make_settings(engine=SQLITE_ENGINE, timeout=60000) + result = merge_statement_timeout(settings) + assert result == {} + + def test_appends_to_existing_options(self): + settings = _make_settings(timeout=60000, existing_options="-c lock_timeout=5000") + with mock.patch.dict('sys.modules', {'uwsgi': None}): + result = merge_statement_timeout(settings) + assert result == {"DATABASES__default__OPTIONS__options": "-c lock_timeout=5000 -c statement_timeout=60000"} diff --git a/awx/settings/__init__.py b/awx/settings/__init__.py index 78afed70bd..6c8f847e72 100644 --- a/awx/settings/__init__.py +++ b/awx/settings/__init__.py @@ -12,6 +12,7 @@ from ansible_base.lib.dynamic_config import ( from .functions import ( assert_production_settings, merge_application_name, + merge_statement_timeout, add_backwards_compatibility, load_extra_development_files, ) @@ -78,6 +79,11 @@ DYNACONF.update( loader_identifier="awx.settings:merge_application_name", merge=True, ) +DYNACONF.update( + merge_statement_timeout(DYNACONF), + loader_identifier="awx.settings:merge_statement_timeout", + merge=True, +) # Update django.conf.settings with DYNACONF values export(__name__, DYNACONF) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index a46ff115c0..58fdd6b51b 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -43,6 +43,11 @@ LISTENER_DATABASES = { } } +# Optional manual override for statement_timeout (ms) on web worker DB +# connections. When running under uwsgi, the timeout is auto-derived from +# the harakiri value. Set this for non-uwsgi deployments or to override. +DATABASE_STATEMENT_TIMEOUT = None + # Whether or not the deployment is a K8S-based deployment # In K8S-based deployments, instances have zero capacity - all playbook # automation is intended to flow through defined Container Groups that diff --git a/awx/settings/functions.py b/awx/settings/functions.py index 70be9befdb..c0a70c4fbd 100644 --- a/awx/settings/functions.py +++ b/awx/settings/functions.py @@ -12,6 +12,41 @@ def merge_application_name(settings): return data +def merge_statement_timeout(settings): + """Return a dynaconf merge dict to set statement_timeout for web worker DB connections. + + Under uwsgi, derives timeout from harakiri with a safety margin so + PostgreSQL cancels the query before uwsgi kills the worker. The margin + is 10% of harakiri, clamped to [1s, 5s]. Falls back to the + DATABASE_STATEMENT_TIMEOUT setting for non-uwsgi deployments. + """ + if "sqlite3" in settings.get("DATABASES__default__ENGINE", ""): + return {} + + timeout_ms = None + try: + import uwsgi + + harakiri = int(uwsgi.opt.get(b'harakiri', 0)) + if harakiri > 0: + margin = min(5, max(1, int(harakiri * 0.1))) + timeout_ms = max(1000, (harakiri - margin) * 1000) + except (ImportError, ValueError): + pass + + if timeout_ms is None: + timeout_ms = settings.get("DATABASE_STATEMENT_TIMEOUT") + + if timeout_ms is None: + return {} + + existing = settings.get("DATABASES__default__OPTIONS__options", "") + new_opt = f"-c statement_timeout={timeout_ms}" + value = f"{existing} {new_opt}".strip() if existing else new_opt + + return {"DATABASES__default__OPTIONS__options": value} + + def add_backwards_compatibility(): """Add backwards compatibility for AWX_MODE.