Recover rsyslog from 4xx error

Due to https://github.com/ansible/awx/issues/7560

'omhttp' module for rsyslog will completely stop forwarding message to external log aggregator after receiving a 4xx error from the external log aggregator

This PR is an "workaround" for this problem by restarting rsyslogd after detecting that rsyslog received a 4xx error
This commit is contained in:
Hao Liu
2023-12-11 16:12:43 -05:00
committed by Hao Liu
parent bf42c63c12
commit 6e5e1c8fff
5 changed files with 25 additions and 23 deletions

View File

@@ -11,12 +11,10 @@ def write_stdout(s):
sys.stdout.write(s)
sys.stdout.flush()
def write_stderr(s):
sys.stderr.write(s)
sys.stderr.flush()
def main():
while 1:
write_stdout("READY\n")
@@ -31,23 +29,6 @@ def main():
except ValueError as e:
write_stderr(str(e))
# now decide what do to based on eventnames
if headers["eventname"] == "PROCESS_STATE_FATAL":
headers.update(
dict(
[x.split(":") for x in sys.stdin.read(int(headers["len"])).split()]
)
)
try:
# incoming event that produced PROCESS_STATE_FATAL will have a PID. SIGTERM it!
write_stderr(
f"{datetime.datetime.now(timezone.utc)} - sending SIGTERM to proc={headers} with data={headers}\n"
)
os.kill(headers["pid"], signal.SIGTERM)
except Exception as e:
write_stderr(str(e))
# awx-rsyslog PROCESS_LOG_STDERR handler
if headers["eventname"] == "PROCESS_LOG_STDERR":
# pertinent data to process that produced PROCES_LOG_STDERR is in the first line of the data payload; so lets extract it