From 42bfff301cbe957726212177fd3553f4be19c4e2 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 22 Mar 2019 08:46:05 -0400 Subject: [PATCH 1/3] remove python-memcached as a base dependency for playbook execution --- docs/custom_virtualenvs.md | 8 ++++---- requirements/requirements_ansible.in | 1 - requirements/requirements_ansible.txt | 3 +-- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/docs/custom_virtualenvs.md b/docs/custom_virtualenvs.md index 196ea94841..65657e43a6 100644 --- a/docs/custom_virtualenvs.md +++ b/docs/custom_virtualenvs.md @@ -26,10 +26,10 @@ the semantics for creating virtualenvs in Python 3 has changed slightly: $ sudo python3 -m venv /var/lib/awx/venv/my-custom-venv Your newly created virtualenv needs a few base dependencies to properly run -playbooks (awx uses memcached as a holding space for playbook artifacts and +playbooks: fact gathering): - $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install python-memcached psutil + $ sudo /var/lib/awx/venv/my-custom-venv/bin/pip install psutil From here, you can install _additional_ Python dependencies that you care about, such as a per-virtualenv version of ansible itself: @@ -61,7 +61,7 @@ index aa8b304..eb05f91 100644 +requirements_custom: + virtualenv $(VENV_BASE)/my-custom-env -+ $(VENV_BASE)/my-custom-env/bin/pip install python-memcached psutil ++ $(VENV_BASE)/my-custom-env/bin/pip install psutil + diff --git a/installer/image_build/templates/Dockerfile.j2 b/installer/image_build/templates/Dockerfile.j2 index d69e2c9..a08bae5 100644 @@ -97,7 +97,7 @@ Now create an initContainer stanza. You can subsititute your own custom images pip install virtualenv && virtualenv /var/lib/awx/venv/my-custom-venv && source /var/lib/awx/venv/my-custom-venv/bin/activate && - /var/lib/awx/venv/my-custom-venv/bin/pip install python-memcached psutil && + /var/lib/awx/venv/my-custom-venv/bin/pip install psutil && /var/lib/awx/venv/my-custom-venv/bin/pip install -U "ansible == X.Y.Z" && /var/lib/awx/venv/my-custom-venv/bin/pip install -U custom-python-module volumeMounts: diff --git a/requirements/requirements_ansible.in b/requirements/requirements_ansible.in index 10e90355e7..2d07079fde 100644 --- a/requirements/requirements_ansible.in +++ b/requirements/requirements_ansible.in @@ -38,7 +38,6 @@ netaddr ovirt-engine-sdk-python==4.2.4 # minimum set inside Ansible facts module requirements # AWX usage pexpect==4.6.0 # same as AWX requirement -python-memcached==1.59 # same as AWX requirement psutil==5.4.3 # same as AWX requirement setuptools==36.0.1 pip==9.0.1 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt index 6a61b4bbd1..dab6a18f1d 100644 --- a/requirements/requirements_ansible.txt +++ b/requirements/requirements_ansible.txt @@ -91,7 +91,6 @@ pynacl==1.2.1 # via paramiko pyopenssl==17.5.0 # via azure-cli-core, pyvmomi, requests-credssp pyparsing==2.2.0 # via packaging python-dateutil==2.6.1 # via adal, azure-storage, botocore -python-memcached==1.59 pyvmomi==6.5 pywinrm[kerberos]==0.3.0 pyyaml==3.12 # via azure-cli-core, knack, openstacksdk, os-client-config @@ -105,7 +104,7 @@ s3transfer==0.1.13 # via boto3 secretstorage==2.3.1 # via keyring selectors2==2.0.1 # via ncclient shade==1.27.0 -six==1.11.0 # via azure-cli-core, bcrypt, cryptography, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, python-memcached, pyvmomi, pywinrm, stevedore +six==1.11.0 # via azure-cli-core, bcrypt, cryptography, isodate, keystoneauth1, knack, munch, ncclient, ntlm-auth, openstacksdk, ovirt-engine-sdk-python, packaging, pynacl, pyopenssl, python-dateutil, pyvmomi, pywinrm, stevedore stevedore==1.28.0 # via keystoneauth1 tabulate==0.7.7 # via azure-cli-core, knack urllib3==1.24 # via requests From e6abd77c963fe7325ef1aaa45d50f51246e871f6 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 22 Mar 2019 08:47:18 -0400 Subject: [PATCH 2/3] remove py2 compatability from awx.main.expect.run this module is no longer run on isolated nodes (they use runner now) --- awx/main/expect/run.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/awx/main/expect/run.py b/awx/main/expect/run.py index f255bbc477..75a6247ec7 100755 --- a/awx/main/expect/run.py +++ b/awx/main/expect/run.py @@ -14,10 +14,7 @@ import signal import sys import threading import time -try: - from io import StringIO -except ImportError: - from StringIO import StringIO +from io import StringIO import pexpect import psutil @@ -231,10 +228,7 @@ def handle_termination(pid, args, proot_cmd, is_cancel=True): instance's cancel_flag. ''' try: - if sys.version_info > (3, 0): - used_proot = proot_cmd.encode('utf-8') in args - else: - used_proot = proot_cmd in ' '.join(args) + used_proot = proot_cmd.encode('utf-8') in args if used_proot: if not psutil: os.kill(pid, signal.SIGKILL) From af8e0718404d09191c876d8dee3d64a2b58a8c33 Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Fri, 22 Mar 2019 10:23:55 -0400 Subject: [PATCH 3/3] remove old callback plugin code and tests --- awx/lib/tests/test_display_callback.py | 353 ---------------------- awx/main/expect/run.py | 11 - awx/main/tasks.py | 3 - awx/main/tests/unit/expect/test_expect.py | 10 +- tools/scripts/awx-expect | 4 - 5 files changed, 1 insertion(+), 380 deletions(-) delete mode 100644 awx/lib/tests/test_display_callback.py delete mode 100755 tools/scripts/awx-expect diff --git a/awx/lib/tests/test_display_callback.py b/awx/lib/tests/test_display_callback.py deleted file mode 100644 index 01ed0bbd74..0000000000 --- a/awx/lib/tests/test_display_callback.py +++ /dev/null @@ -1,353 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# All Rights Reserved - -from __future__ import absolute_import - -from collections import OrderedDict -import json -import os -import shutil -import sys -import tempfile -from unittest import mock - -import pytest - -# ansible uses `ANSIBLE_CALLBACK_PLUGINS` and `ANSIBLE_STDOUT_CALLBACK` to -# discover callback plugins; `ANSIBLE_CALLBACK_PLUGINS` is a list of paths to -# search for a plugin implementation (which should be named `CallbackModule`) -# -# this code modifies the Python path to make our -# `awx.lib.awx_display_callback` callback importable (because `awx.lib` -# itself is not a package) -# -# we use the `awx_display_callback` imports below within this file, but -# Ansible also uses them when it discovers this file in -# `ANSIBLE_CALLBACK_PLUGINS` -CALLBACK = os.path.splitext(os.path.basename(__file__))[0] -PLUGINS = os.path.dirname(__file__) -with mock.patch.dict(os.environ, {'ANSIBLE_STDOUT_CALLBACK': CALLBACK, - 'ANSIBLE_CALLBACK_PLUGINS': PLUGINS}): - from ansible import __version__ as ANSIBLE_VERSION - from ansible.cli.playbook import PlaybookCLI - from ansible.executor.playbook_executor import PlaybookExecutor - from ansible.inventory.manager import InventoryManager - from ansible.parsing.dataloader import DataLoader - from ansible.vars.manager import VariableManager - - # Add awx/lib to sys.path so we can use the plugin - path = os.path.abspath(os.path.join(PLUGINS, '..', '..', 'lib')) - if path not in sys.path: - sys.path.insert(0, path) - - from awx_display_callback import AWXDefaultCallbackModule as CallbackModule # noqa - from awx_display_callback.events import event_context # noqa - - -@pytest.fixture() -def cache(request): - class Cache(OrderedDict): - def set(self, key, value): - self[key] = value - local_cache = Cache() - patch = mock.patch.object(event_context, 'cache', local_cache) - patch.start() - request.addfinalizer(patch.stop) - return local_cache - - -@pytest.fixture() -def executor(tmpdir_factory, request): - playbooks = request.node.callspec.params.get('playbook') - playbook_files = [] - for name, playbook in playbooks.items(): - filename = str(tmpdir_factory.mktemp('data').join(name)) - with open(filename, 'w') as f: - f.write(playbook) - playbook_files.append(filename) - - cli = PlaybookCLI(['', 'playbook.yml']) - cli.parse() - options = cli.parser.parse_args(['-v'])[0] - loader = DataLoader() - variable_manager = VariableManager(loader=loader) - inventory = InventoryManager(loader=loader, sources='localhost,') - variable_manager.set_inventory(inventory) - - return PlaybookExecutor(playbooks=playbook_files, inventory=inventory, - variable_manager=variable_manager, loader=loader, - options=options, passwords={}) - - -@pytest.mark.parametrize('event', {'playbook_on_start', - 'playbook_on_play_start', - 'playbook_on_task_start', 'runner_on_ok', - 'playbook_on_stats'}) -@pytest.mark.parametrize('playbook', [ -{'helloworld.yml': ''' -- name: Hello World Sample - connection: local - hosts: all - gather_facts: no - tasks: - - name: Hello Message - debug: - msg: "Hello World!" -'''}, # noqa -{'results_included.yml': ''' -- name: Run module which generates results list - connection: local - hosts: all - gather_facts: no - vars: - results: ['foo', 'bar'] - tasks: - - name: Generate results list - debug: - var: results -'''}, # noqa -]) -def test_callback_plugin_receives_events(executor, cache, event, playbook): - executor.run() - assert len(cache) - assert event in [task['event'] for task in cache.values()] - - -@pytest.mark.parametrize('playbook', [ -{'no_log_on_ok.yml': ''' -- name: args should not be logged when task-level no_log is set - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo "SENSITIVE" - no_log: true -'''}, # noqa -{'no_log_on_fail.yml': ''' -- name: failed args should not be logged when task-level no_log is set - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo "SENSITIVE" - no_log: true - failed_when: true - ignore_errors: true -'''}, # noqa -{'no_log_on_skip.yml': ''' -- name: skipped task args should be suppressed with no_log - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo "SENSITIVE" - no_log: true - when: false -'''}, # noqa -{'no_log_on_play.yml': ''' -- name: args should not be logged when play-level no_log set - connection: local - hosts: all - gather_facts: no - no_log: true - tasks: - - shell: echo "SENSITIVE" -'''}, # noqa -{'async_no_log.yml': ''' -- name: async task args should suppressed with no_log - connection: local - hosts: all - gather_facts: no - no_log: true - tasks: - - async: 10 - poll: 1 - shell: echo "SENSITIVE" - no_log: true -'''}, # noqa -{'with_items.yml': ''' -- name: with_items tasks should be suppressed with no_log - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo {{ item }} - no_log: true - with_items: [ "SENSITIVE", "SENSITIVE-SKIPPED", "SENSITIVE-FAILED" ] - when: item != "SENSITIVE-SKIPPED" - failed_when: item == "SENSITIVE-FAILED" - ignore_errors: yes -'''}, # noqa, NOTE: with_items will be deprecated in 2.9 -{'loop.yml': ''' -- name: loop tasks should be suppressed with no_log - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo {{ item }} - no_log: true - loop: [ "SENSITIVE", "SENSITIVE-SKIPPED", "SENSITIVE-FAILED" ] - when: item != "SENSITIVE-SKIPPED" - failed_when: item == "SENSITIVE-FAILED" - ignore_errors: yes -'''}, # noqa -]) -def test_callback_plugin_no_log_filters(executor, cache, playbook): - executor.run() - assert len(cache) - assert 'SENSITIVE' not in json.dumps(cache.items()) - - -@pytest.mark.parametrize('playbook', [ -{'no_log_on_ok.yml': ''' -- name: args should not be logged when no_log is set at the task or module level - connection: local - hosts: all - gather_facts: no - tasks: - - shell: echo "PUBLIC" - - shell: echo "PRIVATE" - no_log: true - - uri: url=https://example.org username="PUBLIC" password="PRIVATE" - - copy: content="PRIVATE" dest="/tmp/tmp_no_log" -'''}, # noqa -]) -def test_callback_plugin_task_args_leak(executor, cache, playbook): - executor.run() - events = cache.values() - assert events[0]['event'] == 'playbook_on_start' - assert events[1]['event'] == 'playbook_on_play_start' - - # task 1 - assert events[2]['event'] == 'playbook_on_task_start' - assert events[3]['event'] == 'runner_on_ok' - - # task 2 no_log=True - assert events[4]['event'] == 'playbook_on_task_start' - assert events[5]['event'] == 'runner_on_ok' - assert 'PUBLIC' in json.dumps(cache.items()) - assert 'PRIVATE' not in json.dumps(cache.items()) - # make sure playbook was successful, so all tasks were hit - assert not events[-1]['event_data']['failures'], 'Unexpected playbook execution failure' - - -@pytest.mark.parametrize('playbook', [ -{'loop_with_no_log.yml': ''' -- name: playbook variable should not be overwritten when using no log - connection: local - hosts: all - gather_facts: no - tasks: - - command: "{{ item }}" - register: command_register - no_log: True - with_items: - - "echo helloworld!" - - debug: msg="{{ command_register.results|map(attribute='stdout')|list }}" -'''}, # noqa -]) -def test_callback_plugin_censoring_does_not_overwrite(executor, cache, playbook): - executor.run() - events = cache.values() - assert events[0]['event'] == 'playbook_on_start' - assert events[1]['event'] == 'playbook_on_play_start' - - # task 1 - assert events[2]['event'] == 'playbook_on_task_start' - # Ordering of task and item events may differ randomly - assert set(['runner_on_ok', 'runner_item_on_ok']) == set([data['event'] for data in events[3:5]]) - - # task 2 no_log=True - assert events[5]['event'] == 'playbook_on_task_start' - assert events[6]['event'] == 'runner_on_ok' - assert 'helloworld!' in events[6]['event_data']['res']['msg'] - - -@pytest.mark.parametrize('playbook', [ -{'strip_env_vars.yml': ''' -- name: sensitive environment variables should be stripped from events - connection: local - hosts: all - tasks: - - shell: echo "Hello, World!" -'''}, # noqa -]) -def test_callback_plugin_strips_task_environ_variables(executor, cache, playbook): - executor.run() - assert len(cache) - for event in cache.values(): - assert os.environ['PATH'] not in json.dumps(event) - - -@pytest.mark.parametrize('playbook', [ -{'custom_set_stat.yml': ''' -- name: custom set_stat calls should persist to the local disk so awx can save them - connection: local - hosts: all - tasks: - - set_stats: - data: - foo: "bar" -'''}, # noqa -]) -def test_callback_plugin_saves_custom_stats(executor, cache, playbook): - try: - private_data_dir = tempfile.mkdtemp() - with mock.patch.dict(os.environ, {'AWX_PRIVATE_DATA_DIR': private_data_dir}): - executor.run() - artifacts_path = os.path.join(private_data_dir, 'artifacts', 'custom') - with open(artifacts_path, 'r') as f: - assert json.load(f) == {'foo': 'bar'} - finally: - shutil.rmtree(os.path.join(private_data_dir)) - - -@pytest.mark.parametrize('playbook', [ -{'handle_playbook_on_notify.yml': ''' -- name: handle playbook_on_notify events properly - connection: local - hosts: all - handlers: - - name: my_handler - debug: msg="My Handler" - tasks: - - debug: msg="My Task" - changed_when: true - notify: - - my_handler -'''}, # noqa -]) -@pytest.mark.skipif(ANSIBLE_VERSION < '2.5', reason="v2_playbook_on_notify doesn't work before ansible 2.5") -def test_callback_plugin_records_notify_events(executor, cache, playbook): - executor.run() - assert len(cache) - notify_events = [x[1] for x in cache.items() if x[1]['event'] == 'playbook_on_notify'] - assert len(notify_events) == 1 - assert notify_events[0]['event_data']['handler'] == 'my_handler' - assert notify_events[0]['event_data']['host'] == 'localhost' - assert notify_events[0]['event_data']['task'] == 'debug' - - -@pytest.mark.parametrize('playbook', [ -{'no_log_module_with_var.yml': ''' -- name: ensure that module-level secrets are redacted - connection: local - hosts: all - vars: - - pw: SENSITIVE - tasks: - - uri: - url: https://example.org - user: john-jacob-jingleheimer-schmidt - password: "{{ pw }}" -'''}, # noqa -]) -def test_module_level_no_log(executor, cache, playbook): - # https://github.com/ansible/tower/issues/1101 - # It's possible for `no_log=True` to be defined at the _module_ level, - # e.g., for the URI module password parameter - # This test ensures that we properly redact those - executor.run() - assert len(cache) - assert 'john-jacob-jingleheimer-schmidt' in json.dumps(cache.items()) - assert 'SENSITIVE' not in json.dumps(cache.items()) diff --git a/awx/main/expect/run.py b/awx/main/expect/run.py index 75a6247ec7..744c145688 100755 --- a/awx/main/expect/run.py +++ b/awx/main/expect/run.py @@ -191,18 +191,7 @@ def run_isolated_job(private_data_dir, secrets, logfile=sys.stdout): job_timeout = secrets.get('job_timeout', 10) pexpect_timeout = secrets.get('pexpect_timeout', 5) - # Use local callback directory - callback_dir = os.getenv('AWX_LIB_DIRECTORY') - if callback_dir is None: - raise RuntimeError('Location for callbacks must be specified ' - 'by environment variable AWX_LIB_DIRECTORY.') - env['ANSIBLE_CALLBACK_PLUGINS'] = os.path.join(callback_dir, 'isolated_callbacks') - if 'AD_HOC_COMMAND_ID' in env: - env['ANSIBLE_STDOUT_CALLBACK'] = 'minimal' - else: - env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' env['AWX_ISOLATED_DATA_DIR'] = private_data_dir - env['PYTHONPATH'] = env.get('PYTHONPATH', '') + callback_dir + ':' venv_path = env.get('VIRTUAL_ENV') if venv_path and not os.path.exists(venv_path): diff --git a/awx/main/tasks.py b/awx/main/tasks.py index a343e8f762..ee3629a4ec 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -1354,7 +1354,6 @@ class RunJob(BaseTask): env['MAX_EVENT_RES'] = str(settings.MAX_EVENT_RES_DATA) if not isolated: env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_path - env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' env['AWX_HOST'] = settings.TOWER_URL_BASE # Create a directory for ControlPath sockets that is unique to each @@ -1632,7 +1631,6 @@ class RunProjectUpdate(BaseTask): env['TMP'] = settings.AWX_PROOT_BASE_PATH env['PROJECT_UPDATE_ID'] = str(project_update.pk) env['ANSIBLE_CALLBACK_PLUGINS'] = self.get_path_to('..', 'plugins', 'callback') - env['ANSIBLE_STDOUT_CALLBACK'] = 'awx_display' return env def _build_scm_url_extra_vars(self, project_update, scm_username='', scm_password=''): @@ -2383,7 +2381,6 @@ class RunAdHocCommand(BaseTask): env['INVENTORY_HOSTVARS'] = str(True) env['ANSIBLE_CALLBACK_PLUGINS'] = plugin_dir env['ANSIBLE_LOAD_CALLBACK_PLUGINS'] = '1' - env['ANSIBLE_STDOUT_CALLBACK'] = 'minimal' # Hardcoded by Ansible for ad-hoc commands (either minimal or oneline). env['ANSIBLE_SFTP_BATCH_MODE'] = 'False' # Specify empty SSH args (should disable ControlPersist entirely for diff --git a/awx/main/tests/unit/expect/test_expect.py b/awx/main/tests/unit/expect/test_expect.py index d167c0733d..f9cab27128 100644 --- a/awx/main/tests/unit/expect/test_expect.py +++ b/awx/main/tests/unit/expect/test_expect.py @@ -189,16 +189,11 @@ def test_run_isolated_job(private_data_dir, rsa_key): mgr.build_isolated_job_data() stdout = StringIO() # Mock environment variables for callback module - with mock.patch('os.getenv') as env_mock: - env_mock.return_value = '/path/to/awx/lib' - status, rc = run.run_isolated_job(private_data_dir, secrets, stdout) + status, rc = run.run_isolated_job(private_data_dir, secrets, stdout) assert status == 'successful' assert rc == 0 assert FILENAME in stdout.getvalue() - assert '/path/to/awx/lib' in env['PYTHONPATH'] - assert env['ANSIBLE_STDOUT_CALLBACK'] == 'awx_display' - assert env['ANSIBLE_CALLBACK_PLUGINS'] == '/path/to/awx/lib/isolated_callbacks' assert env['AWX_ISOLATED_DATA_DIR'] == private_data_dir @@ -230,9 +225,6 @@ def test_run_isolated_adhoc_command(private_data_dir, rsa_key): # an ad-hoc command that runs `pwd` should print `private_data_dir` to stdout assert private_data_dir in stdout.getvalue() - assert '/path/to/awx/lib' in env['PYTHONPATH'] - assert env['ANSIBLE_STDOUT_CALLBACK'] == 'minimal' - assert env['ANSIBLE_CALLBACK_PLUGINS'] == '/path/to/awx/lib/isolated_callbacks' assert env['AWX_ISOLATED_DATA_DIR'] == private_data_dir diff --git a/tools/scripts/awx-expect b/tools/scripts/awx-expect deleted file mode 100755 index c04615281e..0000000000 --- a/tools/scripts/awx-expect +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -AWX_LIB=`/var/lib/awx/venv/awx/bin/python -c 'import os, awx; print os.path.dirname(awx.__file__)'` -. /var/lib/awx/venv/awx/bin/activate -exec env AWX_LIB_DIRECTORY=$AWX_LIB/lib /var/lib/awx/venv/awx/bin/python $AWX_LIB/main/expect/run.pyc "$@"