mirror of
https://github.com/ansible/awx.git
synced 2026-08-04 20:10:01 -02:30
* AAP-84057: Set PostgreSQL statement_timeout on web worker DB connections When uwsgi's harakiri kills a worker, the PostgreSQL backend continues running the query indefinitely. These abandoned queries accumulate and create resource contention for all other queries. Add a connection_created signal handler that sets statement_timeout on new DB connections. Under uwsgi, the timeout is auto-derived from the harakiri value (minus 5s margin so PostgreSQL cancels the query before uwsgi kills the worker). Outside uwsgi (task workers, migrations), no timeout is applied. A manual DATABASE_STATEMENT_TIMEOUT setting is available as a fallback for non-uwsgi deployments. * Remove timeout value caching because it brings no significant gains * Use proportional margin between statement_timeout and harakiri timeout * Fix zero-harakiri test to patch fake uwsgi module instead of None * Refactor statement_timeout from signal handler to connection string Move statement_timeout configuration from a connection_created signal handler (extra SQL round-trip per connection) to a dynaconf merge function that sets it via the libpq OPTIONS connection string parameter. This mirrors the existing merge_application_name() pattern and eliminates the SET statement on every new connection. --------- Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
122 lines
4.7 KiB
Python
122 lines
4.7 KiB
Python
import os
|
|
from ansible_base.lib.dynamic_config import load_python_file_with_injected_context
|
|
from dynaconf import Dynaconf
|
|
from .application_name import get_application_name
|
|
|
|
|
|
def merge_application_name(settings):
|
|
"""Return a dynaconf merge dict to set the application name for the connection."""
|
|
data = {}
|
|
if "sqlite3" not in settings.get("DATABASES__default__ENGINE", ""):
|
|
data["DATABASES__default__OPTIONS__application_name"] = get_application_name(settings.get("CLUSTER_HOST_ID"))
|
|
return data
|
|
|
|
|
|
def merge_statement_timeout(settings):
|
|
"""Return a dynaconf merge dict to set statement_timeout for web worker DB connections.
|
|
|
|
Under uwsgi, derives timeout from harakiri with a safety margin so
|
|
PostgreSQL cancels the query before uwsgi kills the worker. The margin
|
|
is 10% of harakiri, clamped to [1s, 5s]. Falls back to the
|
|
DATABASE_STATEMENT_TIMEOUT setting for non-uwsgi deployments.
|
|
"""
|
|
if "sqlite3" in settings.get("DATABASES__default__ENGINE", ""):
|
|
return {}
|
|
|
|
timeout_ms = None
|
|
try:
|
|
import uwsgi
|
|
|
|
harakiri = int(uwsgi.opt.get(b'harakiri', 0))
|
|
if harakiri > 0:
|
|
margin = min(5, max(1, int(harakiri * 0.1)))
|
|
timeout_ms = max(1000, (harakiri - margin) * 1000)
|
|
except (ImportError, ValueError):
|
|
pass
|
|
|
|
if timeout_ms is None:
|
|
timeout_ms = settings.get("DATABASE_STATEMENT_TIMEOUT")
|
|
|
|
if timeout_ms is None:
|
|
return {}
|
|
|
|
existing = settings.get("DATABASES__default__OPTIONS__options", "")
|
|
new_opt = f"-c statement_timeout={timeout_ms}"
|
|
value = f"{existing} {new_opt}".strip() if existing else new_opt
|
|
|
|
return {"DATABASES__default__OPTIONS__options": value}
|
|
|
|
|
|
def add_backwards_compatibility():
|
|
"""Add backwards compatibility for AWX_MODE.
|
|
|
|
Before dynaconf integration the usage of AWX settings was supported to be just
|
|
DJANGO_SETTINGS_MODULE=awx.settings.production or DJANGO_SETTINGS_MODULE=awx.settings.development
|
|
(development_quiet and development_kube were also supported).
|
|
|
|
With dynaconf the DJANGO_SETTINGS_MODULE should be set always to "awx.settings" as the only entry point
|
|
for settings and then "AWX_MODE" can be set to any of production,development,quiet,kube
|
|
or a combination of them separated by comma.
|
|
|
|
E.g:
|
|
|
|
export DJANGO_SETTINGS_MODULE=awx.settings
|
|
export AWX_MODE=production
|
|
awx-manage [command]
|
|
dynaconf [command]
|
|
|
|
If pointing `DJANGO_SETTINGS_MODULE` to `awx.settings.production` or `awx.settings.development` then
|
|
this function will set `AWX_MODE` to the correct value.
|
|
"""
|
|
django_settings_module = os.getenv("DJANGO_SETTINGS_MODULE", "awx.settings")
|
|
if django_settings_module == "awx.settings":
|
|
return
|
|
|
|
current_mode = os.getenv("AWX_MODE", "")
|
|
for _module_name in ["development", "production", "development_quiet", "development_kube"]:
|
|
if django_settings_module == f"awx.settings.{_module_name}":
|
|
_mode = current_mode.split(",")
|
|
if "development_" in _module_name and "development" not in current_mode:
|
|
_mode.append("development")
|
|
_mode_fragment = _module_name.replace("development_", "")
|
|
if _mode_fragment not in _mode:
|
|
_mode.append(_mode_fragment)
|
|
os.environ["AWX_MODE"] = ",".join(_mode)
|
|
|
|
|
|
def load_extra_development_files(settings: Dynaconf):
|
|
"""Load optional development only settings files."""
|
|
if not settings.is_development_mode:
|
|
return
|
|
|
|
if settings.get_environ("AWX_KUBE_DEVEL"):
|
|
load_python_file_with_injected_context("kube_defaults.py", settings=settings)
|
|
else:
|
|
load_python_file_with_injected_context("local_*.py", settings=settings)
|
|
|
|
|
|
def assert_production_settings(settings: Dynaconf, settings_dir: str, settings_file_path: str): # pragma: no cover
|
|
"""Ensure at least one setting file has been loaded in production mode.
|
|
Current systems will require /etc/tower/settings.py and
|
|
new systems will require /etc/ansible-automation-platform/*.yaml
|
|
"""
|
|
if "production" not in settings.current_env.lower():
|
|
return
|
|
|
|
required_settings_paths = [
|
|
os.path.dirname(settings_file_path),
|
|
"/etc/ansible-automation-platform/",
|
|
settings_dir,
|
|
]
|
|
|
|
for path in required_settings_paths:
|
|
if any([path in os.path.dirname(f) for f in settings._loaded_files]):
|
|
break
|
|
else:
|
|
from django.core.exceptions import ImproperlyConfigured # noqa
|
|
|
|
msg = 'No AWX configuration found at %s.' % required_settings_paths
|
|
msg += '\nDefine the AWX_SETTINGS_FILE environment variable to '
|
|
msg += 'specify an alternate path.'
|
|
raise ImproperlyConfigured(msg)
|