From cd4d83acb7870971c83875735ecc7843fd189580 Mon Sep 17 00:00:00 2001 From: John Westcott IV Date: Mon, 12 Jun 2023 15:29:55 -0400 Subject: [PATCH] Compensating for NUL unicode characters NUL characters are not allowed in text fields in the database We used to strip them out of stdout but the exception changed And we want to be sure to strip them out of JSONBlob fields --- awx/main/dispatch/worker/callback.py | 4 +++- awx/main/fields.py | 18 +++++++++++++++--- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/awx/main/dispatch/worker/callback.py b/awx/main/dispatch/worker/callback.py index e1db0f27e9..8dca8dd894 100644 --- a/awx/main/dispatch/worker/callback.py +++ b/awx/main/dispatch/worker/callback.py @@ -191,7 +191,9 @@ class CallbackBrokerWorker(BaseWorker): e._retry_count = retry_count # special sanitization logic for postgres treatment of NUL 0x00 char - if (retry_count == 1) and isinstance(exc_indv, ValueError) and ("\x00" in e.stdout): + # This used to check the class of the exception but on the postgres3 upgrade it could appear + # as either DataError or ValueError, so now lets just try if its there. + if (retry_count == 1) and ("\x00" in e.stdout): e.stdout = e.stdout.replace("\x00", "") if retry_count >= self.INDIVIDUAL_EVENT_RETRIES: diff --git a/awx/main/fields.py b/awx/main/fields.py index ebe3cfc125..f3b99e2e70 100644 --- a/awx/main/fields.py +++ b/awx/main/fields.py @@ -102,10 +102,22 @@ class JSONBlob(JSONField): if not prepared: value = self.get_prep_value(value) try: - return json.dumps(value) + # Null characters are not allowed in text fields and JSONBlobs are JSON data but saved as text + # So we want to make sure we strip out any null characters also note, these "should" be escaped by the dumps process: + # >>> my_obj = { 'test': '\x00' } + # >>> import json + # >>> json.dumps(my_obj) + # '{"test": "\\u0000"}' + # But just to be safe, lets remove them if they are there. \x00 and \u0000 are the same: + # >>> string = "\x00" + # >>> "\u0000" in string + # True + dumped_value = json.dumps(value) + if "\x00" in dumped_value: + dumped_value = dumped_value.replace("\x00", '') + return dumped_value except Exception as e: - logger.error(f"Failed to dump JSONField {self.name}: {e}") - logger.error(f"{value}") + logger.error(f"Failed to dump JSONField {self.name}: {e} value: {value}") return value