remove python log rotation in favor of system

* The cron ran logrotate will now rotate our log files instead of python
* If not error log file is specified in the config then do not include
it as a paremter to rsyslog omhttp module. This is useful for
containers.

awx/main/tests/unit/api/test_logger.py

@@ -35,16 +35,17 @@ data_loggly = {
 # Test reconfigure logging settings function
 # name this whatever you want
 @pytest.mark.parametrize(
-    'enabled, log_type, host, port, protocol, expected_config', [
+    'enabled, log_type, host, port, protocol, errorfile, expected_config', [
         (
             True,
             'loggly',
             'http://logs-01.loggly.com/inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/',
             None,
             'https',
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
+                'action(type="omhttp" server="logs-01.loggly.com" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="inputs/1fd38090-2af1-4e1e-8d80-492899da0f71/tag/http/")',  # noqa
             ])
         ),
         (
@@ -53,6 +54,7 @@ data_loggly = {
             'localhost',
             9000,
             'udp',
+            '',  # empty errorfile
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")',
                 'action(type="omfwd" target="localhost" port="9000" protocol="udp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx")',  # noqa
@@ -64,6 +66,7 @@ data_loggly = {
             'localhost',
             9000,
             'tcp',
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")',
                 'action(type="omfwd" target="localhost" port="9000" protocol="tcp" action.resumeRetryCount="-1" action.resumeInterval="5" template="awx")',  # noqa
@@ -75,9 +78,10 @@ data_loggly = {
             'https://yoursplunk/services/collector/event',
             None,
             None,
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -86,9 +90,10 @@ data_loggly = {
             'http://yoursplunk/services/collector/event',
             None,
             None,
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="80" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -97,9 +102,10 @@ data_loggly = {
             'https://yoursplunk:8088/services/collector/event',
             None,
             None,
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -108,9 +114,10 @@ data_loggly = {
             'https://yoursplunk/services/collector/event',
             8088,
             None,
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -119,9 +126,10 @@ data_loggly = {
             'yoursplunk.org/services/collector/event',
             8088,
             'https',
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -130,9 +138,10 @@ data_loggly = {
             'http://yoursplunk.org/services/collector/event',
             8088,
             None,
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="services/collector/event")',  # noqa
+                'action(type="omhttp" server="yoursplunk.org" serverport="8088" usehttps="off" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="services/collector/event")',  # noqa
             ])
         ),
         (
@@ -141,14 +150,15 @@ data_loggly = {
             'https://endpoint5.collection.us2.sumologic.com/receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==', # noqa
             None,
             'https',
+            '/var/log/tower/rsyslog.err',
             '\n'.join([
                 'template(name="awx" type="string" string="%rawmsg-after-pri%")\nmodule(load="omhttp")',
-                'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" errorfile="/var/log/tower/rsyslog.err" action.resumeInterval="5" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")',  # noqa
+                'action(type="omhttp" server="endpoint5.collection.us2.sumologic.com" serverport="443" usehttps="on" allowunsignedcerts="off" skipverifyhost="off" action.resumeRetryCount="-1" template="awx" action.resumeInterval="5" errorfile="/var/log/tower/rsyslog.err" restpath="receiver/v1/http/ZaVnC4dhaV0qoiETY0MrM3wwLoDgO1jFgjOxE6-39qokkj3LGtOroZ8wNaN2M6DtgYrJZsmSi4-36_Up5TbbN_8hosYonLKHSSOSKY845LuLZBCBwStrHQ==")',  # noqa
             ])
         ),
     ]
 )
-def test_rsyslog_conf_template(enabled, log_type, host, port, protocol, expected_config):
+def test_rsyslog_conf_template(enabled, log_type, host, port, protocol, errorfile, expected_config):
     
     mock_settings, _ = _mock_logging_defaults()
     
@@ -159,6 +169,7 @@ def test_rsyslog_conf_template(enabled, log_type, host, port, protocol, expected
     setattr(mock_settings, 'LOG_AGGREGATOR_ENABLED', enabled)
     setattr(mock_settings, 'LOG_AGGREGATOR_TYPE', log_type)
     setattr(mock_settings, 'LOG_AGGREGATOR_HOST', host)
+    setattr(mock_settings, 'LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE', errorfile)
     if port:
         setattr(mock_settings, 'LOG_AGGREGATOR_PORT', port)
     if protocol:

awx/main/utils/external_logging.py

@@ -18,6 +18,7 @@ def construct_rsyslog_conf_template(settings=settings):
     timeout = getattr(settings, 'LOG_AGGREGATOR_TCP_TIMEOUT', 5)
     max_disk_space = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_GB', 1)
     spool_directory = getattr(settings, 'LOG_AGGREGATOR_MAX_DISK_USAGE_PATH', '/var/lib/awx').rstrip('/')
+    error_log_file = getattr(settings, 'LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE', '')
 
     if not os.access(spool_directory, os.W_OK):
         spool_directory = '/var/lib/awx'
@@ -74,9 +75,10 @@ def construct_rsyslog_conf_template(settings=settings):
             f'skipverifyhost="{skip_verify}"',
             'action.resumeRetryCount="-1"',
             'template="awx"',
-            'errorfile="/var/log/tower/rsyslog.err"',
             f'action.resumeInterval="{timeout}"'
         ]
+        if error_log_file:
+            params.append(f'errorfile="{error_log_file}"')
         if parsed.path:
             path = urlparse.quote(parsed.path[1:], safe='/=')
             if parsed.query:

awx/settings/defaults.py

@@ -770,6 +770,7 @@ LOG_AGGREGATOR_LEVEL = 'INFO'
 LOG_AGGREGATOR_MAX_DISK_USAGE_GB = 1
 LOG_AGGREGATOR_MAX_DISK_USAGE_PATH = '/var/lib/awx'
 LOG_AGGREGATOR_RSYSLOGD_DEBUG = False
+LOG_AGGREGATOR_RSYSLOGD_ERROR_LOG_FILE = '/var/log/tower/rsyslog.err'
 
 # The number of retry attempts for websocket session establishment
 # If you're encountering issues establishing websockets in clustered Tower,
@@ -853,38 +854,30 @@ LOGGING = {
         },
         'tower_warnings': {
             # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false', 'dynamic_level_filter'],
             'filename': os.path.join(LOG_ROOT, 'tower.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'callback_receiver': {
             # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false', 'dynamic_level_filter'],
             'filename': os.path.join(LOG_ROOT, 'callback_receiver.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'dispatcher': {
             # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false', 'dynamic_level_filter'],
             'filename': os.path.join(LOG_ROOT, 'dispatcher.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'dispatcher',
         },
         'wsbroadcast': {
             # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false', 'dynamic_level_filter'],
             'filename': os.path.join(LOG_ROOT, 'wsbroadcast.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'celery.beat': {
@@ -898,46 +891,36 @@ LOGGING = {
         },
         'task_system': {
             # don't define a level here, it's set by settings.LOG_AGGREGATOR_LEVEL
-            'class': 'logging.handlers.RotatingFileHandler',
+            'class': 'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false', 'dynamic_level_filter'],
             'filename': os.path.join(LOG_ROOT, 'task_system.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'management_playbooks': {
             'level': 'DEBUG',
-            'class':'logging.handlers.RotatingFileHandler',
+            'class':'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false'],
             'filename': os.path.join(LOG_ROOT, 'management_playbooks.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'system_tracking_migrations': {
             'level': 'WARNING',
-            'class':'logging.handlers.RotatingFileHandler',
+            'class':'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false'],
             'filename': os.path.join(LOG_ROOT, 'tower_system_tracking_migrations.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'rbac_migrations': {
             'level': 'WARNING',
-            'class':'logging.handlers.RotatingFileHandler',
+            'class':'logging.handlers.WatchedFileHandler',
             'filters': ['require_debug_false'],
             'filename': os.path.join(LOG_ROOT, 'tower_rbac_migrations.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
         'isolated_manager': {
             'level': 'WARNING',
-            'class':'logging.handlers.RotatingFileHandler',
+            'class':'logging.handlers.WatchedFileHandler',
             'filename': os.path.join(LOG_ROOT, 'isolated_manager.log'),
-            'maxBytes': 1024 * 1024 * 5, # 5 MB
-            'backupCount': 5,
             'formatter':'simple',
         },
     },