diff --git a/awx/main/conf.py b/awx/main/conf.py index 767d5a251f..e11bb15ae4 100644 --- a/awx/main/conf.py +++ b/awx/main/conf.py @@ -694,16 +694,18 @@ register( category_slug='logging', ) register( - 'LOG_AGGREGATOR_MAX_DISK_USAGE_GB', + 'LOG_AGGREGATOR_ACTION_QUEUE_SIZE', field_class=fields.IntegerField, - default=1, + default=131072, min_value=1, - label=_('Maximum disk persistence for external log aggregation (in GB)'), + label=_('Maximum number of messages that can be stored in the log action queue'), help_text=_( - 'Amount of data to store (in gigabytes) during an outage of ' - 'the external log aggregator (defaults to 1). ' - 'Equivalent to the rsyslogd queue.maxdiskspace setting for main_queue. ' - 'Notably, this is used for the rsyslogd main queue (for input messages).' + 'Defines how large the rsyslog action queue can grow in number of messages ' + 'stored. This can have an impact on memory utilization. When the queue ' + 'reaches 75% of this number, the queue will start writing to disk ' + '(queue.highWatermark in rsyslog). When it reaches 90%, NOTICE, INFO, and ' + 'DEBUG messages will start to be discarded (queue.discardMark with ' + 'queue.discardSeverity=5).' ), category=_('Logging'), category_slug='logging', @@ -718,8 +720,7 @@ register( 'Amount of data to store (in gigabytes) if an rsyslog action takes time ' 'to process an incoming message (defaults to 1). ' 'Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). ' - 'Like LOG_AGGREGATOR_MAX_DISK_USAGE_GB, it stores files in the directory specified ' - 'by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.' + 'It stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.' ), category=_('Logging'), category_slug='logging', diff --git a/awx/main/tests/unit/api/test_logger.py b/awx/main/tests/unit/api/test_logger.py index 5bc20aef1b..a3da0d23b8 100644 --- a/awx/main/tests/unit/api/test_logger.py +++ b/awx/main/tests/unit/api/test_logger.py @@ -47,7 +47,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")', # noqa + 'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")', # noqa ] ), ), @@ -61,7 +61,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")', - 'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx")', # noqa + 'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5")', # noqa ] ), ), @@ -75,7 +75,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")', - 'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx")', # noqa + 'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5")', # noqa ] ), ), @@ -89,7 +89,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -103,7 +103,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -117,7 +117,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -131,7 +131,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -145,7 +145,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -159,7 +159,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa + 'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")', # noqa ] ), ), @@ -173,7 +173,7 @@ data_loggly = { '\n'.join( [ 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")', - 'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxdiskspace="1g" queue.type="LinkedList" queue.saveOnShutdown="on" errorfile="/var/log/tower/rsyslog.err" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")', # noqa + 'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" queue.spoolDirectory="/var/lib/awx" queue.filename="awx-external-logger-action-queue" queue.maxDiskSpace="1g" queue.maxFileSize="100m" queue.type="LinkedList" queue.saveOnShutdown="on" queue.syncqueuefiles="on" queue.checkpointInterval="1000" queue.size="131072" queue.highwaterMark="98304" queue.discardMark="117964" queue.discardSeverity="5" errorfile="/var/log/tower/rsyslog.err" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")', # noqa ] ), ), diff --git a/awx/main/utils/external_logging.py b/awx/main/utils/external_logging.py index e97e80a3ad..061ab40a95 100644 --- a/awx/main/utils/external_logging.py +++ b/awx/main/utils/external_logging.py @@ -17,11 +17,26 @@ def construct_rsyslog_conf_template(settings=settings): port = getattr(settings, 'LOG_AGGREGATOR_PORT', '') protocol = getattr(settings, 'LOG_AGGREGATOR_PROTOCOL', '') timeout = getattr(settings, 'LOG_AGGREGATOR_TCP_TIMEOUT', 5) - max_disk_space_main_queue = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_GB', 1) + action_queue_size = getattr(settings, 'LOG_AGGREGATOR_ACTION_QUEUE_SIZE', 131072) max_disk_space_action_queue = getattr(settings, 'LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB', 1) spool_directory = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_PATH', '/var/lib/awx').rstrip('/') error_log_file = getattr(settings, 'LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE', '') + queue_options = [ + f'queue.spoolDirectory="{spool_directory}"', + 'queue.filename="awx-external-logger-action-queue"', + f'queue.maxDiskSpace="{max_disk_space_action_queue}g"', # overall disk space for all queue files + 'queue.maxFileSize="100m"', # individual file size + 'queue.type="LinkedList"', + 'queue.saveOnShutdown="on"', + 'queue.syncqueuefiles="on"', # (f)sync when checkpoint occurs + 'queue.checkpointInterval="1000"', # Update disk queue every 1000 messages + f'queue.size="{action_queue_size}"', # max number of messages in queue + f'queue.highwaterMark="{int(action_queue_size * 0.75)}"', # 75% of queue.size + f'queue.discardMark="{int(action_queue_size * 0.9)}"', # 90% of queue.size + 'queue.discardSeverity="5"', # Only discard notice, info, debug if we must discard anything + ] + if not os.access(spool_directory, os.W_OK): spool_directory = '/var/lib/awx' @@ -33,7 +48,6 @@ def construct_rsyslog_conf_template(settings=settings): '$WorkDirectory /var/lib/awx/rsyslog', f'$MaxMessageSize {max_bytes}', '$IncludeConfig /var/lib/awx/rsyslog/conf.d/*.conf', - f'main_queue(queue.spoolDirectory="{spool_directory}" queue.maxdiskspace="{max_disk_space_main_queue}g" queue.type="Disk" queue.filename="awx-external-logger-backlog")', # noqa 'module(load="imuxsock" SysSock.Use="off")', 'input(type="imuxsock" Socket="' + settings.LOGGING['handlers']['external_logger']['address'] + '" unlink="on" RateLimit.Burst="0")', 'template(name="awx" type="string" string="%rawmsg-after-pri%")', @@ -79,12 +93,7 @@ def construct_rsyslog_conf_template(settings=settings): 'action.resumeRetryCount="-1"', 'template="awx"', f'action.resumeInterval="{timeout}"', - f'queue.spoolDirectory="{spool_directory}"', - 'queue.filename="awx-external-logger-action-queue"', - f'queue.maxdiskspace="{max_disk_space_action_queue}g"', - 'queue.type="LinkedList"', - 'queue.saveOnShutdown="on"', - ] + ] + queue_options if error_log_file: params.append(f'errorfile="{error_log_file}"') if parsed.path: @@ -112,9 +121,18 @@ def construct_rsyslog_conf_template(settings=settings): params = ' '.join(params) parts.extend(['module(load="omhttp")', f'action({params})']) elif protocol and host and port: - parts.append( - f'action(type="omfwd" target="{host}" port="{port}" protocol="{protocol}" action.resumeRetryCount="-1" action.resumeInterval="{timeout}" template="awx")' # noqa - ) + params = [ + 'type="omfwd"', + f'target="{host}"', + f'port="{port}"', + f'protocol="{protocol}"', + 'action.resumeRetryCount="-1"', + f'action.resumeInterval="{timeout}"', + 'template="awx"', + ] + queue_options + params = ' '.join(params) + parts.append(f'action({params})') + else: parts.append('action(type="omfile" file="/dev/null")') # rsyslog needs *at least* one valid action to start tmpl = '\n'.join(parts) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index 19a83cdbcc..d12fa8d223 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -796,7 +796,7 @@ LOG_AGGREGATOR_ENABLED = False LOG_AGGREGATOR_TCP_TIMEOUT = 5 LOG_AGGREGATOR_VERIFY_CERT = True LOG_AGGREGATOR_LEVEL = 'INFO' -LOG_AGGREGATOR_MAX_DISK_USAGE_GB = 1 # Main queue +LOG_AGGREGATOR_ACTION_QUEUE_SIZE = 131072 LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB = 1 # Action queue LOG_AGGREGATOR_MAX_DISK_USAGE_PATH = '/var/lib/awx' LOG_AGGREGATOR_RSYSLOGD_DEBUG = False diff --git a/awx/ui/src/screens/Setting/Logging/Logging.test.js b/awx/ui/src/screens/Setting/Logging/Logging.test.js index 4f5428e739..decfb00d7d 100644 --- a/awx/ui/src/screens/Setting/Logging/Logging.test.js +++ b/awx/ui/src/screens/Setting/Logging/Logging.test.js @@ -29,7 +29,7 @@ SettingsAPI.readCategory.mockResolvedValue({ LOG_AGGREGATOR_TCP_TIMEOUT: 5, LOG_AGGREGATOR_VERIFY_CERT: true, LOG_AGGREGATOR_LEVEL: 'INFO', - LOG_AGGREGATOR_MAX_DISK_USAGE_GB: 1, + LOG_AGGREGATOR_ACTION_QUEUE_SIZE: 131072, LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB: 1, LOG_AGGREGATOR_MAX_DISK_USAGE_PATH: '/var/lib/awx', LOG_AGGREGATOR_RSYSLOGD_DEBUG: false, diff --git a/awx/ui/src/screens/Setting/Logging/LoggingEdit/LoggingEdit.test.js b/awx/ui/src/screens/Setting/Logging/LoggingEdit/LoggingEdit.test.js index 55d87ef2ce..4f088decc6 100644 --- a/awx/ui/src/screens/Setting/Logging/LoggingEdit/LoggingEdit.test.js +++ b/awx/ui/src/screens/Setting/Logging/LoggingEdit/LoggingEdit.test.js @@ -31,7 +31,7 @@ const mockSettings = { LOG_AGGREGATOR_TCP_TIMEOUT: 123, LOG_AGGREGATOR_VERIFY_CERT: true, LOG_AGGREGATOR_LEVEL: 'ERROR', - LOG_AGGREGATOR_MAX_DISK_USAGE_GB: 1, + LOG_AGGREGATOR_ACTION_QUEUE_SIZE: 131072, LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB: 1, LOG_AGGREGATOR_MAX_DISK_USAGE_PATH: '/var/lib/awx', LOG_AGGREGATOR_RSYSLOGD_DEBUG: false, diff --git a/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json b/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json index 10c398d94b..d544bcf427 100644 --- a/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json +++ b/awx/ui/src/screens/Setting/shared/data.allSettingOptions.json @@ -659,21 +659,21 @@ ] ] }, - "LOG_AGGREGATOR_MAX_DISK_USAGE_GB": { + "LOG_AGGREGATOR_ACTION_QUEUE_SIZE": { "type": "integer", "required": false, - "label": "Maximum disk persistence for external log aggregation (in GB)", - "help_text": "Amount of data to store (in gigabytes) during an outage of the external log aggregator (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting for main_queue. Notably, this is used for the rsyslogd main queue (for input messages).", + "label": "Maximum number of messages that can be stored in the log action queue", + "help_text": "Defines how large the rsyslog action queue can grow in number of messages stored. This can have an impact on memory utilization. When the queue reaches 75% of this number, the queue will start writing to disk (queue.highWatermark in rsyslog). When it reaches 90%, NOTICE, INFO, and DEBUG messages will start to be discarded (queue.discardMark with queue.discardSeverity=5).", "min_value": 1, "category": "Logging", "category_slug": "logging", - "default": 1 + "default": 131072 }, "LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB": { "type": "integer", "required": false, "label": "Maximum disk persistence for rsyslogd action queuing (in GB)", - "help_text": "Amount of data to store (in gigabytes) if an rsyslog action takes time to process an incoming message (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). Like LOG_AGGREGATOR_MAX_DISK_USAGE_GB, it stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.", + "help_text": "Amount of data to store (in gigabytes) if an rsyslog action takes time to process an incoming message (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). It stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.", "min_value": 1, "category": "Logging", "category_slug": "logging", @@ -5016,10 +5016,10 @@ ] ] }, - "LOG_AGGREGATOR_MAX_DISK_USAGE_GB": { + "LOG_AGGREGATOR_ACTION_QUEUE_SIZE": { "type": "integer", - "label": "Maximum disk persistence for external log aggregation (in GB)", - "help_text": "Amount of data to store (in gigabytes) during an outage of the external log aggregator (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting for main_queue. Notably, this is used for the rsyslogd main queue (for input messages).", + "label": "Maximum number of messages that can be stored in the log action queue", + "help_text": "Defines how large the rsyslog action queue can grow in number of messages stored. This can have an impact on memory utilization. When the queue reaches 75% of this number, the queue will start writing to disk (queue.highWatermark in rsyslog). When it reaches 90%, NOTICE, INFO, and DEBUG messages will start to be discarded (queue.discardMark with queue.discardSeverity=5).", "min_value": 1, "category": "Logging", "category_slug": "logging", @@ -5028,7 +5028,7 @@ "LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB": { "type": "integer", "label": "Maximum disk persistence for rsyslogd action queuing (in GB)", - "help_text": "Amount of data to store (in gigabytes) if an rsyslog action takes time to process an incoming message (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). Like LOG_AGGREGATOR_MAX_DISK_USAGE_GB, it stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.", + "help_text": "Amount of data to store (in gigabytes) if an rsyslog action takes time to process an incoming message (defaults to 1). Equivalent to the rsyslogd queue.maxdiskspace setting on the action (e.g. omhttp). It stores files in the directory specified by LOG_AGGREGATOR_MAX_DISK_USAGE_PATH.", "min_value": 1, "category": "Logging", "category_slug": "logging", diff --git a/awx/ui/src/screens/Setting/shared/data.allSettings.json b/awx/ui/src/screens/Setting/shared/data.allSettings.json index 7a67b71c6d..37d560537a 100644 --- a/awx/ui/src/screens/Setting/shared/data.allSettings.json +++ b/awx/ui/src/screens/Setting/shared/data.allSettings.json @@ -70,7 +70,7 @@ "LOG_AGGREGATOR_TCP_TIMEOUT": 5, "LOG_AGGREGATOR_VERIFY_CERT": true, "LOG_AGGREGATOR_LEVEL": "INFO", - "LOG_AGGREGATOR_MAX_DISK_USAGE_GB": 1, + "LOG_AGGREGATOR_ACTION_QUEUE_SIZE": 131072, "LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB": 1, "LOG_AGGREGATOR_MAX_DISK_USAGE_PATH": "/var/lib/awx", "LOG_AGGREGATOR_RSYSLOGD_DEBUG": false, @@ -548,4 +548,4 @@ "adj_list": [] } } -} \ No newline at end of file +} diff --git a/awx/ui/src/screens/Setting/shared/data.logSettings.json b/awx/ui/src/screens/Setting/shared/data.logSettings.json index 65a0fa8915..e97c3ce8fc 100644 --- a/awx/ui/src/screens/Setting/shared/data.logSettings.json +++ b/awx/ui/src/screens/Setting/shared/data.logSettings.json @@ -15,7 +15,7 @@ "LOG_AGGREGATOR_TCP_TIMEOUT": 5, "LOG_AGGREGATOR_VERIFY_CERT": true, "LOG_AGGREGATOR_LEVEL": "INFO", - "LOG_AGGREGATOR_MAX_DISK_USAGE_GB": 1, + "LOG_AGGREGATOR_ACTION_QUEUE_SIZE": 131072, "LOG_AGGREGATOR_ACTION_MAX_DISK_USAGE_GB": 1, "LOG_AGGREGATOR_MAX_DISK_USAGE_PATH": "/var/lib/awx", "LOG_AGGREGATOR_RSYSLOGD_DEBUG": false,