Merge pull request #9883 from shanemcd/remove-resource-profiling

Remove resource profiling feature

Reviewed-by: Ryan Petrello <ryan@ryanpetrello.com>
Reviewed-by: Jake McDermott <yo@jakemcdermott.me>

awx/main/conf.py

@@ -6,7 +6,6 @@ from django.utils.translation import ugettext_lazy as _
 
 # Django REST Framework
 from rest_framework import serializers
-from rest_framework.fields import FloatField
 
 # Tower
 from awx.conf import fields, register, register_validate
@@ -344,49 +343,6 @@ register(
     category_slug='jobs',
 )
 
-register(
-    'AWX_RESOURCE_PROFILING_ENABLED',
-    field_class=fields.BooleanField,
-    default=False,
-    label=_('Enable detailed resource profiling on all playbook runs'),
-    help_text=_('If set, detailed resource profiling data will be collected on all jobs. ' 'This data can be gathered with `sosreport`.'),  # noqa
-    category=_('Jobs'),
-    category_slug='jobs',
-)
-
-register(
-    'AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL',
-    field_class=FloatField,
-    default='0.25',
-    label=_('Interval (in seconds) between polls for cpu usage.'),
-    help_text=_('Interval (in seconds) between polls for cpu usage. ' 'Setting this lower than the default will affect playbook performance.'),
-    category=_('Jobs'),
-    category_slug='jobs',
-    required=False,
-)
-
-register(
-    'AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL',
-    field_class=FloatField,
-    default='0.25',
-    label=_('Interval (in seconds) between polls for memory usage.'),
-    help_text=_('Interval (in seconds) between polls for memory usage. ' 'Setting this lower than the default will affect playbook performance.'),
-    category=_('Jobs'),
-    category_slug='jobs',
-    required=False,
-)
-
-register(
-    'AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL',
-    field_class=FloatField,
-    default='0.25',
-    label=_('Interval (in seconds) between polls for PID count.'),
-    help_text=_('Interval (in seconds) between polls for PID count. ' 'Setting this lower than the default will affect playbook performance.'),
-    category=_('Jobs'),
-    category_slug='jobs',
-    required=False,
-)
-
 register(
     'AWX_TASK_ENV',
     field_class=fields.KeyValueField,

awx/main/tasks.py

@@ -991,36 +991,6 @@ class BaseTask(object):
         Build ansible yaml file filled with extra vars to be passed via -e@file.yml
         """
 
-    def build_params_resource_profiling(self, instance, private_data_dir):
-        resource_profiling_params = {}
-        if self.should_use_resource_profiling(instance):
-            cpu_poll_interval = settings.AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL
-            mem_poll_interval = settings.AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL
-            pid_poll_interval = settings.AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL
-
-            results_dir = os.path.join(private_data_dir, 'artifacts/playbook_profiling')
-            if not os.path.isdir(results_dir):
-                os.makedirs(results_dir, stat.S_IREAD | stat.S_IWRITE | stat.S_IEXEC)
-            # FIXME: develop some better means of referencing paths inside containers
-            container_results_dir = os.path.join('/runner', 'artifacts/playbook_profiling')
-
-            logger.debug(
-                'Collected the following resource profiling intervals: cpu: {} mem: {} pid: {}'.format(cpu_poll_interval, mem_poll_interval, pid_poll_interval)
-            )
-
-            resource_profiling_params.update(
-                {
-                    'resource_profiling': True,
-                    'resource_profiling_base_cgroup': 'ansible-runner',
-                    'resource_profiling_cpu_poll_interval': cpu_poll_interval,
-                    'resource_profiling_memory_poll_interval': mem_poll_interval,
-                    'resource_profiling_pid_poll_interval': pid_poll_interval,
-                    'resource_profiling_results_dir': container_results_dir,
-                }
-            )
-
-        return resource_profiling_params
-
     def _write_extra_vars_file(self, private_data_dir, vars, safe_dict={}):
         env_path = os.path.join(private_data_dir, 'env')
         try:
@@ -1088,12 +1058,6 @@ class BaseTask(object):
 
         return env
 
-    def should_use_resource_profiling(self, job):
-        """
-        Return whether this task should use resource profiling
-        """
-        return False
-
     def build_inventory(self, instance, private_data_dir):
         script_params = dict(hostvars=True, towervars=True)
         if hasattr(instance, 'job_slice_number'):
@@ -1395,7 +1359,6 @@ class BaseTask(object):
             passwords = self.build_passwords(self.instance, kwargs)
             self.build_extra_vars_file(self.instance, private_data_dir)
             args = self.build_args(self.instance, private_data_dir, passwords)
-            resource_profiling_params = self.build_params_resource_profiling(self.instance, private_data_dir)
             env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files)
             self.safe_env = build_safe_env(env)
 
@@ -1422,7 +1385,6 @@ class BaseTask(object):
                 'settings': {
                     'job_timeout': self.get_instance_timeout(self.instance),
                     'suppress_ansible_output': True,
-                    **resource_profiling_params,
                 },
             }
 
@@ -1766,12 +1728,6 @@ class RunJob(BaseTask):
                 d[r'Vault password \({}\):\s*?$'.format(vault_id)] = k
         return d
 
-    def should_use_resource_profiling(self, job):
-        """
-        Return whether this task should use resource profiling
-        """
-        return settings.AWX_RESOURCE_PROFILING_ENABLED
-
     def build_execution_environment_params(self, instance, private_data_dir):
         if settings.IS_K8S:
             return {}

awx/main/tests/unit/test_tasks.py

@@ -543,32 +543,6 @@ class TestGenericRun:
             1, job_args=json.dumps({'foo': 'bar'}), job_cwd='/foobar', job_env={'switch': 'blade', 'foot': 'ball', 'secret_key': 'redacted_value'}
         )
 
-    @mock.patch('os.makedirs')
-    def test_build_params_resource_profiling(self, os_makedirs):
-        job = Job(project=Project(), inventory=Inventory())
-        task = tasks.RunJob()
-        task.should_use_resource_profiling = lambda job: True
-        task.instance = job
-
-        resource_profiling_params = task.build_params_resource_profiling(task.instance, '/fake_private_data_dir')
-        assert resource_profiling_params['resource_profiling'] is True
-        assert resource_profiling_params['resource_profiling_base_cgroup'] == 'ansible-runner'
-        assert resource_profiling_params['resource_profiling_cpu_poll_interval'] == '0.25'
-        assert resource_profiling_params['resource_profiling_memory_poll_interval'] == '0.25'
-        assert resource_profiling_params['resource_profiling_pid_poll_interval'] == '0.25'
-        assert resource_profiling_params['resource_profiling_results_dir'] == '/runner/artifacts/playbook_profiling'
-
-    @pytest.mark.parametrize("scenario, profiling_enabled", [('global_setting', True), ('default', False)])
-    def test_should_use_resource_profiling(self, scenario, profiling_enabled, settings):
-        job = Job(project=Project(), inventory=Inventory())
-        task = tasks.RunJob()
-        task.instance = job
-
-        if scenario == 'global_setting':
-            settings.AWX_RESOURCE_PROFILING_ENABLED = True
-
-        assert task.should_use_resource_profiling(task.instance) == profiling_enabled
-
     def test_created_by_extra_vars(self):
         job = Job(created_by=User(pk=123, username='angry-spud'))
 

awx/settings/defaults.py

@@ -592,18 +592,6 @@ AWX_ISOLATION_SHOW_PATHS = []
 # Note: This setting may be overridden by database settings.
 AWX_ISOLATION_BASE_PATH = "/tmp"
 
-# Disable resource profiling by default
-AWX_RESOURCE_PROFILING_ENABLED = False
-
-# Interval (in seconds) between polls for cpu usage
-AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL = '0.25'
-
-# Interval (in seconds) between polls for memory usage
-AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL = '0.25'
-
-# Interval (in seconds) between polls for PID count
-AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL = '0.25'
-
 # User definable ansible callback plugins
 # Note: This setting may be overridden by database settings.
 AWX_ANSIBLE_CALLBACK_PLUGINS = ""

awx/ui_next/src/screens/Setting/Jobs/JobsDetail/JobsDetail.test.jsx

@@ -55,11 +55,6 @@ describe('<JobsDetail />', () => {
     assertDetail(wrapper, 'Isolated launch timeout', '600 seconds');
     assertDetail(wrapper, 'Isolated connection timeout', '10 seconds');
     assertDetail(wrapper, 'Isolated host key checking', 'Off');
-    assertDetail(
-      wrapper,
-      'Enable detailed resource profiling on all playbook runs',
-      'Off'
-    );
     assertDetail(wrapper, 'Run Project Updates With Higher Verbosity', 'Off');
     assertDetail(wrapper, 'Enable Role Download', 'On');
     assertDetail(wrapper, 'Enable Collection(s) Download', 'On');

awx/ui_next/src/screens/Setting/Jobs/JobsEdit/JobsEdit.jsx

@@ -174,10 +174,6 @@ function JobsEdit() {
                     name="AWX_ISOLATED_HOST_KEY_CHECKING"
                     config={jobs.AWX_ISOLATED_HOST_KEY_CHECKING}
                   />
-                  <BooleanField
-                    name="AWX_RESOURCE_PROFILING_ENABLED"
-                    config={jobs.AWX_RESOURCE_PROFILING_ENABLED}
-                  />
                   <InputField
                     name="AWX_ISOLATED_CHECK_INTERVAL"
                     config={jobs.AWX_ISOLATED_CHECK_INTERVAL}

awx/ui_next/src/screens/Setting/Jobs/JobsEdit/data.defaultJobSettings.json

@@ -29,10 +29,6 @@
   "AWX_ISOLATED_LAUNCH_TIMEOUT": 600,
   "AWX_ISOLATION_BASE_PATH": "/tmp",
   "AWX_ISOLATION_SHOW_PATHS": [],
-  "AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL": 0.25,
-  "AWX_RESOURCE_PROFILING_ENABLED": false,
-  "AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL": 0.25,
-  "AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL": 0.25,
   "AWX_ROLES_ENABLED": true,
   "AWX_SHOW_PLAYBOOK_LINKS": false,
   "AWX_TASK_ENV": {},

awx/ui_next/src/screens/Setting/shared/data.allSettingOptions.json

@@ -247,38 +247,6 @@
               "category_slug": "jobs",
               "defined_in_file": false
           },
-          "AWX_RESOURCE_PROFILING_ENABLED": {
-              "type": "boolean",
-              "label": "Enable detailed resource profiling on all playbook runs",
-              "help_text": "If set, detailed resource profiling data will be collected on all jobs. This data can be gathered with `sosreport`.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "defined_in_file": false
-          },
-          "AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL": {
-              "type": "float",
-              "label": "Interval (in seconds) between polls for cpu usage.",
-              "help_text": "Interval (in seconds) between polls for cpu usage. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "defined_in_file": false
-          },
-          "AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL": {
-              "type": "float",
-              "label": "Interval (in seconds) between polls for memory usage.",
-              "help_text": "Interval (in seconds) between polls for memory usage. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "defined_in_file": false
-          },
-          "AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL": {
-              "type": "float",
-              "label": "Interval (in seconds) between polls for PID count.",
-              "help_text": "Interval (in seconds) between polls for PID count. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "defined_in_file": false
-          },
           "AWX_TASK_ENV": {
               "type": "nested object",
               "label": "Extra Environment Variables",
@@ -3212,42 +3180,6 @@
               "category_slug": "jobs",
               "default": false
           },
-          "AWX_RESOURCE_PROFILING_ENABLED": {
-              "type": "boolean",
-              "required": false,
-              "label": "Enable detailed resource profiling on all playbook runs",
-              "help_text": "If set, detailed resource profiling data will be collected on all jobs. This data can be gathered with `sosreport`.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "default": false
-          },
-          "AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL": {
-              "type": "float",
-              "required": false,
-              "label": "Interval (in seconds) between polls for cpu usage.",
-              "help_text": "Interval (in seconds) between polls for cpu usage. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "default": 0.25
-          },
-          "AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL": {
-              "type": "float",
-              "required": false,
-              "label": "Interval (in seconds) between polls for memory usage.",
-              "help_text": "Interval (in seconds) between polls for memory usage. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "default": 0.25
-          },
-          "AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL": {
-              "type": "float",
-              "required": false,
-              "label": "Interval (in seconds) between polls for PID count.",
-              "help_text": "Interval (in seconds) between polls for PID count. Setting this lower than the default will affect playbook performance.",
-              "category": "Jobs",
-              "category_slug": "jobs",
-              "default": 0.25
-          },
           "AWX_TASK_ENV": {
               "type": "nested object",
               "required": false,

awx/ui_next/src/screens/Setting/shared/data.allSettings.json

@@ -43,10 +43,6 @@
   "AWX_ISOLATED_KEY_GENERATION":true,
   "AWX_ISOLATED_PRIVATE_KEY":"",
   "AWX_ISOLATED_PUBLIC_KEY":"",
-  "AWX_RESOURCE_PROFILING_ENABLED":false,
-  "AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL":0.25,
-  "AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL":0.25,
-  "AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL":0.25,
   "AWX_TASK_ENV":{},
   "INSIGHTS_TRACKING_STATE":false,
   "PROJECT_UPDATE_VVV":false,

awx/ui_next/src/screens/Setting/shared/data.jobSettings.json

@@ -13,10 +13,6 @@
   "AWX_ISOLATED_KEY_GENERATION": true,
   "AWX_ISOLATED_PRIVATE_KEY": "",
   "AWX_ISOLATED_PUBLIC_KEY": "",
-  "AWX_RESOURCE_PROFILING_ENABLED": false,
-  "AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL": 0.25,
-  "AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL": 0.25,
-  "AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL": 0.25,
   "AWX_TASK_ENV": {},
   "PROJECT_UPDATE_VVV": false,
   "AWX_ROLES_ENABLED": true,

docs/performance_data.md

@@ -1,18 +0,0 @@
-Performance Data
-================
-
-AWX has the ability to collect performance data on job runs.
-
-The following data is collected periodically (with a default interval of every 0.25 seconds):
-* CPU usage
-* Memory usage
-* PID count
-
-The data is stored under `/var/log/tower/playbook_profiling`. A new folder is created for each job run. The folder's name is set to the job's ID.
-
-Performance data collection is not enabled by default. To enable performance data collection on all jobs, set AWX_RESOURCE_PROFILING_ENABLED to true.
-
-The frequency with which data is collected can be set using:
-* AWX_RESOURCE_PROFILING_CPU_POLL_INTERVAL
-* AWX_RESOURCE_PROFILING_MEMORY_POLL_INTERVAL
-* AWX_RESOURCE_PROFILING_PID_POLL_INTERVAL