From 264c508c805267b420a56a8429d1293ac4355ef0 Mon Sep 17 00:00:00 2001 From: Alan Rominger Date: Tue, 1 Mar 2022 20:08:54 -0500 Subject: [PATCH] Move model and settings operations out of threaded code This is to avoid references to settings in threads, this is known to create problems when caches expire this leads to KeyError in environments with heavy load --- awx/main/tasks/receptor.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/awx/main/tasks/receptor.py b/awx/main/tasks/receptor.py index d2809e27ac..5d58217b6f 100644 --- a/awx/main/tasks/receptor.py +++ b/awx/main/tasks/receptor.py @@ -286,19 +286,18 @@ class AWXReceptorJob: # reading. sockin, sockout = socket.socketpair() + # Prepare the submit_work kwargs before creating threads, because references to settings are not thread-safe + work_submit_kw = dict(worktype=self.work_type, params=self.receptor_params, signwork=self.sign_work) + if self.work_type == 'ansible-runner': + work_submit_kw['node'] = self.task.instance.execution_node + use_stream_tls = get_conn_type(work_submit_kw['node'], receptor_ctl).name == "STREAMTLS" + work_submit_kw['tlsclient'] = get_tls_client(use_stream_tls) + with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: transmitter_future = executor.submit(self.transmit, sockin) - _kw = {} - if self.work_type == 'ansible-runner': - _kw['node'] = self.task.instance.execution_node - use_stream_tls = get_conn_type(_kw['node'], receptor_ctl).name == "STREAMTLS" - _kw['tlsclient'] = get_tls_client(use_stream_tls) - # submit our work, passing in the right side of our socketpair for reading. - result = receptor_ctl.submit_work( - worktype=self.work_type, payload=sockout.makefile('rb'), params=self.receptor_params, signwork=self.sign_work, **_kw - ) + result = receptor_ctl.submit_work(payload=sockout.makefile('rb'), **work_submit_kw) sockin.close() sockout.close()