FIX: Refactor formatted raw SQL in unified_jobs result_stdout_raw_handle (#16522)

* Refactor result_stdout_raw_handle to use parameterized COPY SQL.
Replace f-string SQL construction with psycopg.sql composables and bound
parameters so security scans no longer flag formatted raw SQL in the
unified jobs stdout path.
Fix sqlite_copy mock rendering for psycopg3 SQL composables.

* Fix sqlite_copy mock without psycopg SQL internals.
Load stdout from the first populated event table instead of rendering
psycopg composables, which use version-specific private attributes.

* Use sql.Literal in COPY query for Django cursor.copy compatibility.
Django's cursor.copy() does not forward bind parameters to psycopg,
which caused stdout API 500s against real PostgreSQL.

.tekton/run-atf-tests-pull-request.yaml

@@ -18,7 +18,7 @@ metadata:
     pipelines.appstudio.openshift.io/type: build
 spec:
   timeouts:
-    pipeline: "9h"
+    pipeline: "8h"
     tasks: "7h"
     finally: "1h"
   pipelineRef:

awx/main/db/profiled_pg/base.py

@@ -68,7 +68,7 @@ class RecordedQueryLog(object):
                     progname = match
                     break
             else:
-                progname = os.path.basename(sys.argv[0])
+                progname = 'unknown'
             filepath = os.path.join(self.dest, '{}.sqlite'.format(progname))
             version = _get_version('awx')
             log = sqlite3.connect(filepath, timeout=3)

awx/main/models/unified_jobs.py

@@ -20,6 +20,9 @@ from dispatcherd.factories import get_control_from_settings
 # Django
 from django.conf import settings
 from django.db import models, connection, transaction
+
+# psycopg
+from psycopg import sql
 from django.db.models.constraints import UniqueConstraint
 from django.core.exceptions import NON_FIELD_ERRORS
 from django.utils.translation import gettext_lazy as _
@@ -1179,17 +1182,23 @@ class UnifiedJob(
                         raise StdoutMaxBytesExceeded(total, max_supported)
 
                 tbl = self._meta.db_table + 'event'
-                created_by_cond = ''
+                where_parts = [
+                    sql.SQL('{} = {}').format(sql.Identifier(self.event_parent_key), sql.Literal(self.id)),
+                    sql.SQL("stdout != ''"),
+                ]
                 if self.has_unpartitioned_events:
-                    tbl = f'_unpartitioned_{tbl}'
+                    tbl = '_unpartitioned_' + tbl
                 else:
-                    created_by_cond = f"job_created='{self.created.isoformat()}' AND "
+                    where_parts.insert(0, sql.SQL('job_created = {}').format(sql.Literal(self.created)))
 
-                sql = f"copy (select stdout from {tbl} where {created_by_cond}{self.event_parent_key}={self.id} and stdout != '' order by start_line) to stdout"  # nosql
+                copy_sql = sql.SQL('COPY (SELECT stdout FROM {} WHERE {} ORDER BY start_line) TO STDOUT').format(
+                    sql.Identifier(tbl),
+                    sql.SQL(' AND ').join(where_parts),
+                )
                 # psycopg3's copy writes bytes, but callers of this
                 # function assume a str-based fd will be returned; decode
                 # .write() calls on the fly to maintain this interface
-                with cursor.copy(sql) as copy:
+                with cursor.copy(copy_sql) as copy:
                     while data := copy.read():
                         fd.write(smart_str(bytes(data)))
 

awx/main/tests/functional/conftest.py

@@ -830,14 +830,13 @@ class MockCopy:
     events = []
     index = -1
 
-    def __init__(self, sql):
+    def __init__(self):
         self.events = []
-        parts = sql.split(' ')
-        tablename = parts[parts.index('from') + 1]
         for cls in (JobEvent, AdHocCommandEvent, ProjectUpdateEvent, InventoryUpdateEvent, SystemJobEvent):
-            if cls._meta.db_table == tablename:
-                for event in cls.objects.order_by('start_line').all():
-                    self.events.append(event.stdout)
+            events = list(cls.objects.order_by('start_line').values_list('stdout', flat=True))
+            if events:
+                self.events = events
+                break
 
     def read(self):
         self.index = self.index + 1
@@ -858,9 +857,8 @@ def sqlite_copy(request, mocker):
     # copy is postgres-specific, and SQLite doesn't support it; mock its
     # behavior to test that it writes a file that contains stdout from events
 
-    def write_stdout(self, sql):
-        mock_copy = MockCopy(sql)
-        return mock_copy
+    def write_stdout(self, sql, params=None):
+        return MockCopy()
 
     mocker.patch.object(SQLiteCursorWrapper, 'copy', write_stdout, create=True)
 

awx/main/tests/unit/test_db.py

@@ -1,7 +1,6 @@
 import collections
 import os
 import sqlite3
-import sys
 import unittest
 
 import pytest
@@ -125,7 +124,7 @@ def test_sql_above_threshold(tmpdir):
         args, kw = _call
         assert args == ('EXPLAIN VERBOSE {}'.format(QUERY['sql']),)
 
-    path = os.path.join(tmpdir, '{}.sqlite'.format(os.path.basename(sys.argv[0])))
+    path = os.path.join(tmpdir, 'unknown.sqlite')
     assert os.path.exists(path)
 
     # verify the results

awx_collection/plugins/module_utils/controller_api.py

@@ -438,7 +438,7 @@ class ControllerAPIModule(ControllerModule):
             raise RuntimeError('Expected list from API at {0}, got: {1}'.format(endpoint, response))
         next_page = response['json']['next']
 
-        if response['json']['count'] > 10000:
+        if response['json'].get('count', 0) > 10000:
             self.fail_json(msg='The number of items being queried for is higher than 10,000.')
 
         while next_page is not None:
@@ -493,8 +493,11 @@ class ControllerAPIModule(ControllerModule):
                     fail_msg += ', detail: {0}'.format(response['json']['detail'])
                 self.fail_json(msg=fail_msg)
 
-            if 'count' not in response['json'] or 'results' not in response['json']:
-                self.fail_json(msg="The endpoint did not provide count and results")
+            if 'results' not in response['json']:
+                self.fail_json(msg="The endpoint did not provide a results list")
+
+            if 'count' not in response['json']:
+                response['json']['count'] = len(response['json']['results'])
 
         if response['json']['count'] == 0:
             if allow_none:

awxkit/conftest.py

@@ -0,0 +1,16 @@
+# This conftest registers pytest-django CLI options and INI keys as harmless
+# no-ops so that the awxkit tox environment (which does not install Django or
+# pytest-django) does not crash when the root pytest.ini config leaks through.
+#
+# Root pytest.ini contains settings like --reuse-db, --nomigrations, and
+# DJANGO_SETTINGS_MODULE that are only meaningful for Django tests.  Placing
+# this conftest at the awxkit/ level ensures it is loaded before argument
+# parsing regardless of which config file pytest ultimately selects.
+
+
+def pytest_addoption(parser):
+    group = parser.getgroup("awxkit-compat", "awxkit tox compatibility shims")
+    group.addoption("--reuse-db", action="store_true", default=False, help="(no-op in awxkit) pytest-django compat")
+    group.addoption("--nomigrations", action="store_true", default=False, help="(no-op in awxkit) pytest-django compat")
+    group.addoption("--create-db", action="store_true", default=False, help="(no-op in awxkit) pytest-django compat")
+    parser.addini("DJANGO_SETTINGS_MODULE", help="(no-op in awxkit) pytest-django compat", default="")

awxkit/pytest.ini

@@ -0,0 +1,9 @@
+[pytest]
+addopts = -v --tb=native
+testpaths = test
+python_files = test_*.py
+
+filterwarnings =
+  error
+
+junit_family=xunit2

awxkit/tox.ini

@@ -19,7 +19,7 @@ deps =
     pytest-mock
 
 commands =
-    coverage run --parallel --source awxkit -m pytest --doctest-glob='*.md' --junit-xml=report.xml {posargs}
+    coverage run --parallel --source awxkit -m pytest -c pytest.ini test --doctest-glob='*.md' --junit-xml=report.xml {posargs}
     coverage combine
     coverage xml
 
@@ -44,6 +44,8 @@ max-line-length = 120
 
 [pytest]
 addopts = -v --tb=native
+testpaths = test
+python_files = test_*.py
 
 filterwarnings =
   error