feat: remove extra_vars from jobs and unified_jobs list endpoint. Add include query parameter.

awx/api/serializers.py

@@ -961,14 +961,32 @@ class UnifiedJobSerializer(BaseSerializer):
 
 
 class UnifiedJobListSerializer(UnifiedJobSerializer):
+    # these fields can be included optionally in the response
+    OPTIONAL_INCLUDE_FIELDS = frozenset({'artifacts', 'extra_vars'})
+
+    # these fields are stripped from the response
+    _STRIPPED_FIELDS = frozenset({'job_args', 'job_cwd', 'job_env', 'result_traceback', 'event_processing_finished', 'artifacts', 'extra_vars'})
+
     class Meta:
-        fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback', '-event_processing_finished', '-artifacts')
+        fields = ('*', '-job_args', '-job_cwd', '-job_env', '-result_traceback', '-event_processing_finished', '-artifacts', '-extra_vars')
+
+    # processes the include query param if present
+    def _requested_includes(self):
+        request = self.context.get('request')
+        if request is None:
+            return frozenset()
+        raw = request.query_params.get('include', '')
+        requested = {name.strip() for name in raw.split(',') if name.strip()}
+
+        # only allow the fields listed in OPTIONAL_INCLUDE_FIELDS
+        return frozenset(requested) & self.OPTIONAL_INCLUDE_FIELDS
 
     def get_field_names(self, declared_fields, info):
         field_names = super(UnifiedJobListSerializer, self).get_field_names(declared_fields, info)
         # Meta multiple inheritance and -field_name options don't seem to be
         # taking effect above, so remove the undesired fields here.
-        return tuple(x for x in field_names if x not in ('job_args', 'job_cwd', 'job_env', 'result_traceback', 'event_processing_finished', 'artifacts'))
+        strip = self._STRIPPED_FIELDS - self._requested_includes()
+        return tuple(x for x in field_names if x not in strip)
 
     def get_types(self):
         if type(self) is UnifiedJobListSerializer:

awx/api/views/__init__.py

@@ -127,6 +127,7 @@ from awx.api.views.mixin import (
     RelatedJobsPreventDeleteMixin,
     UnifiedJobDeletionMixin,
     NoTruncateMixin,
+    UnifiedJobIncludeMixin,
 )
 from awx.api.pagination import UnifiedJobEventPagination
 from awx.main.utils import set_environ
@@ -3850,7 +3851,7 @@ class SystemJobTemplateNotificationTemplatesSuccessList(SystemJobTemplateNotific
     resource_purpose = 'notification templates triggered on system job success'
 
 
-class JobList(ListAPIView):
+class JobList(UnifiedJobIncludeMixin, ListAPIView):
     model = models.Job
     serializer_class = serializers.JobListSerializer
     resource_purpose = 'jobs'
@@ -4567,7 +4568,7 @@ class UnifiedJobTemplateList(ListAPIView):
     resource_purpose = 'unified job templates'
 
 
-class UnifiedJobList(ListAPIView):
+class UnifiedJobList(UnifiedJobIncludeMixin, ListAPIView):
     model = models.UnifiedJob
     serializer_class = serializers.UnifiedJobListSerializer
     search_fields = ('description', 'name', 'job__playbook')

awx/api/views/mixin.py

@@ -212,3 +212,9 @@ class NoTruncateMixin(object):
         if self.request.query_params.get('no_truncate'):
             context.update(no_truncate=True)
         return context
+
+
+class UnifiedJobIncludeMixin(object):
+    # Reserve the name 'include' so we can use it as a query param. Otherwise, the rest-filters backend
+    # would treat it as a model field lookup.
+    rest_filters_reserved_names = ('include',)

awx/main/tests/functional/api/test_unified_jobs_view.py

@@ -145,3 +145,124 @@ def test_delete_ad_hoc_command_in_active_state(ad_hoc_command_factory, delete, a
     adhoc = ad_hoc_command_factory(initial_state=status)
     url = reverse('api:ad_hoc_command_detail', kwargs={'pk': adhoc.pk})
     delete(url, None, admin, expect=403)
+
+
+@pytest.fixture
+def job_with_heavy_fields(job_factory):
+    job = job_factory()
+    job.extra_vars = '{"some_var": "some_value"}'
+    job.artifacts = {"some_artifact": "some_value"}
+    job.save()
+    return job
+
+
+def _job_result(response, job_id):
+    for row in response.data['results']:
+        if row['id'] == job_id:
+            return row
+    raise AssertionError('job {} not found in {}'.format(job_id, [r['id'] for r in response.data['results']]))
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_strips_heavy_fields_by_default(get, admin, job_with_heavy_fields):
+    response = get(reverse('api:unified_job_list') + '?id={}'.format(job_with_heavy_fields.id), admin, expect=200)
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' not in row
+    assert 'extra_vars' not in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_artifacts(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=artifacts'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' in row
+    assert 'extra_vars' not in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_extra_vars(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=extra_vars'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'extra_vars' in row
+    assert 'artifacts' not in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_both(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=artifacts,extra_vars'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' in row
+    assert 'extra_vars' in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_tolerates_whitespace(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=%20artifacts%20,%20extra_vars%20'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' in row
+    assert 'extra_vars' in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_ignores_unknown(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=does_not_exist'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' not in row
+    assert 'extra_vars' not in row
+
+
+@pytest.mark.django_db
+def test_unified_jobs_list_include_does_not_honor_disallowed(get, admin, job_with_heavy_fields):
+    # event_processing_finished triggers a count(*) on main_jobevent and must
+    # not be re-enabled via the public ?include= param.
+    response = get(
+        reverse('api:unified_job_list') + '?id={}&include=event_processing_finished,job_args,result_traceback'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'event_processing_finished' not in row
+    assert 'job_args' not in row
+    assert 'result_traceback' not in row
+    assert 'artifacts' not in row
+    assert 'extra_vars' not in row
+
+
+@pytest.mark.django_db
+def test_jobs_list_strips_heavy_fields_by_default(get, admin, job_with_heavy_fields):
+    response = get(reverse('api:job_list') + '?id={}'.format(job_with_heavy_fields.id), admin, expect=200)
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'artifacts' not in row
+    assert 'extra_vars' not in row
+
+
+@pytest.mark.django_db
+def test_jobs_list_include_extra_vars(get, admin, job_with_heavy_fields):
+    response = get(
+        reverse('api:job_list') + '?id={}&include=extra_vars'.format(job_with_heavy_fields.id),
+        admin,
+        expect=200,
+    )
+    row = _job_result(response, job_with_heavy_fields.id)
+    assert 'extra_vars' in row
+    assert 'artifacts' not in row

awx/main/tests/unit/api/serializers/test_unified_serializers.py

@@ -39,7 +39,7 @@ def test_unified_job_detail_exclusive_fields():
     For each type, assert that the only fields allowed to be exclusive to
     detail view are the allowed types
     """
-    allowed_detail_fields = frozenset(('result_traceback', 'job_args', 'job_cwd', 'job_env', 'event_processing_finished', 'artifacts'))
+    allowed_detail_fields = frozenset(('result_traceback', 'job_args', 'job_cwd', 'job_env', 'event_processing_finished', 'artifacts', 'extra_vars'))
     for cls in UnifiedJob.__subclasses__():
         list_serializer = getattr(serializers, '{}ListSerializer'.format(cls.__name__))
         detail_serializer = getattr(serializers, '{}Serializer'.format(cls.__name__))