improve cleanup of anonymous kubeconfig files

awx/main/isolated/manager.py

@@ -6,6 +6,7 @@ import stat
 import tempfile
 import time
 import logging
+import yaml
 
 from django.conf import settings
 import ansible_runner
@@ -48,10 +49,17 @@ class IsolatedManager(object):
     def build_inventory(self, hosts):
         if self.instance and self.instance.is_containerized:
             inventory = {'all': {'hosts': {}}}
+            fd, path = tempfile.mkstemp(
+                prefix='.kubeconfig', dir=self.private_data_dir
+            )
+            with open(path, 'wb') as temp:
+                temp.write(yaml.dump(self.pod_manager.kube_config).encode())
+                temp.flush()
+                os.chmod(temp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
             for host in hosts:
                 inventory['all']['hosts'][host] = {
                     "ansible_connection": "kubectl",
-                    "ansible_kubectl_config": self.pod_manager.kube_config
+                    "ansible_kubectl_config": path,
                 }
         else:
             inventory = '\n'.join([
@@ -143,6 +151,8 @@ class IsolatedManager(object):
             '- /artifacts/job_events/*-partial.json.tmp',
             # don't rsync the ssh_key FIFO
             '- /env/ssh_key',
+            # don't rsync kube config files
+            '- .kubeconfig*'
         ]
 
         for filename, data in (

awx/main/scheduler/kubernetes.py

@@ -1,9 +1,5 @@
 import collections
-import os
-import stat
 import time
-import yaml
-import tempfile
 import logging
 from base64 import b64encode
 
@@ -88,8 +84,17 @@ class PodManager(object):
 
     @cached_property
     def kube_api(self):
-        my_client = config.new_client_from_config(config_file=self.kube_config)
+        # this feels a little janky, but it's what k8s' own code does
-        return client.CoreV1Api(api_client=my_client)
+        # internally when it reads kube config files from disk:
+        # https://github.com/kubernetes-client/python-base/blob/0b208334ef0247aad9afcaae8003954423b61a0d/config/kube_config.py#L643
+        loader = config.kube_config.KubeConfigLoader(
+            config_dict=self.kube_config
+        )
+        cfg = type.__call__(client.Configuration)
+        loader.load_and_set(cfg)
+        return client.CoreV1Api(api_client=client.ApiClient(
+            configuration=cfg
+        ))
 
     @property
     def pod_name(self):
@@ -174,10 +179,4 @@ def generate_tmp_kube_config(credential, namespace):
         ).decode() # decode the base64 data into a str
     else:
         config["clusters"][0]["cluster"]["insecure-skip-tls-verify"] = True
-
+    return config
-    fd, path = tempfile.mkstemp(prefix='kubeconfig')
-    with open(path, 'wb') as temp:
-        temp.write(yaml.dump(config).encode())
-        temp.flush()
-        os.chmod(temp.name, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
-    return path

awx/main/tasks.py

@@ -1423,7 +1423,6 @@ class BaseTask(object):
     def deploy_container_group_pod(self, task):
         from awx.main.scheduler.kubernetes import PodManager # Avoid circular import
         pod_manager = PodManager(self.instance)
-        self.cleanup_paths.append(pod_manager.kube_config)
         try:
             log_name = task.log_format
             logger.debug(f"Launching pod for {log_name}.")

awx/main/tests/functional/task_management/test_container_groups.py

@@ -1,5 +1,4 @@
 import subprocess
-import yaml
 import base64
 
 from unittest import mock # noqa
@@ -51,6 +50,5 @@ def test_kubectl_ssl_verification(containerized_job):
     cred.inputs['ssl_ca_cert'] = cert.stdout
     cred.save()
     pm = PodManager(containerized_job)
-    config = yaml.load(open(pm.kube_config), Loader=yaml.FullLoader)
+    ca_data = pm.kube_config['clusters'][0]['cluster']['certificate-authority-data']
-    ca_data = config['clusters'][0]['cluster']['certificate-authority-data']
     assert cert.stdout == base64.b64decode(ca_data.encode())