diff --git a/awx/main/analytics/collectors.py b/awx/main/analytics/collectors.py index e7fc7a8c90..ab59db5983 100644 --- a/awx/main/analytics/collectors.py +++ b/awx/main/analytics/collectors.py @@ -220,17 +220,11 @@ def projects_by_scm_type(since, **kwargs): return counts -def _get_isolated_datetime(last_check): - if last_check: - return last_check.isoformat() - return last_check - - -@register('instance_info', '1.0', description=_('Cluster topology and capacity')) +@register('instance_info', '1.1', description=_('Cluster topology and capacity')) def instance_info(since, include_hostnames=False, **kwargs): info = {} instances = models.Instance.objects.values_list('hostname').values( - 'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'last_isolated_check', 'enabled' + 'uuid', 'version', 'capacity', 'cpu', 'memory', 'managed_by_policy', 'hostname', 'enabled' ) for instance in instances: consumed_capacity = sum(x.task_impact for x in models.UnifiedJob.objects.filter(execution_node=instance['hostname'], status__in=('running', 'waiting'))) @@ -241,7 +235,6 @@ def instance_info(since, include_hostnames=False, **kwargs): 'cpu': instance['cpu'], 'memory': instance['memory'], 'managed_by_policy': instance['managed_by_policy'], - 'last_isolated_check': _get_isolated_datetime(instance['last_isolated_check']), 'enabled': instance['enabled'], 'consumed_capacity': consumed_capacity, 'remaining_capacity': instance['capacity'] - consumed_capacity, diff --git a/awx/main/analytics/metrics.py b/awx/main/analytics/metrics.py index 0af34a60ea..03ea674db8 100644 --- a/awx/main/analytics/metrics.py +++ b/awx/main/analytics/metrics.py @@ -184,7 +184,6 @@ def metrics(): INSTANCE_INFO.labels(hostname=hostname, instance_uuid=uuid).info( { 'enabled': str(instance_data[uuid]['enabled']), - 'last_isolated_check': getattr(instance_data[uuid], 'last_isolated_check', 'None'), 'managed_by_policy': str(instance_data[uuid]['managed_by_policy']), 'version': instance_data[uuid]['version'], } diff --git a/awx/main/isolated/.gitignore b/awx/main/isolated/.gitignore deleted file mode 100644 index 05b023b41d..0000000000 --- a/awx/main/isolated/.gitignore +++ /dev/null @@ -1 +0,0 @@ -authorized_keys diff --git a/awx/main/isolated/__init__.py b/awx/main/isolated/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/awx/main/isolated/manager.py b/awx/main/isolated/manager.py deleted file mode 100644 index 3fbda06ab8..0000000000 --- a/awx/main/isolated/manager.py +++ /dev/null @@ -1,365 +0,0 @@ -import fnmatch -import json -import os -import shutil -import stat -import tempfile -import time -import logging -import datetime - -from django.conf import settings -import ansible_runner - -import awx -from awx.main.utils import get_system_task_capacity - -logger = logging.getLogger('awx.isolated.manager') -playbook_logger = logging.getLogger('awx.isolated.manager.playbooks') - - -def set_pythonpath(venv_libdir, env): - env.pop('PYTHONPATH', None) # default to none if no python_ver matches - for version in os.listdir(venv_libdir): - if fnmatch.fnmatch(version, 'python[23].*'): - if os.path.isdir(os.path.join(venv_libdir, version)): - env['PYTHONPATH'] = os.path.join(venv_libdir, version, "site-packages") + ":" - break - - -class IsolatedManager(object): - def __init__(self, event_handler, canceled_callback=None, check_callback=None): - """ - :param event_handler: a callable used to persist event data from isolated nodes - :param canceled_callback: a callable - which returns `True` or `False` - - signifying if the job has been prematurely - canceled - """ - self.event_handler = event_handler - self.canceled_callback = canceled_callback - self.check_callback = check_callback - self.started_at = None - self.captured_command_artifact = False - self.instance = None - - def build_inventory(self, hosts): - inventory = '\n'.join(['{} ansible_ssh_user={}'.format(host, settings.AWX_ISOLATED_USERNAME) for host in hosts]) - - return inventory - - def build_runner_params(self, hosts, verbosity=1): - env = dict(os.environ.items()) - env['ANSIBLE_RETRY_FILES_ENABLED'] = 'False' - env['ANSIBLE_HOST_KEY_CHECKING'] = str(settings.AWX_ISOLATED_HOST_KEY_CHECKING) - env['ANSIBLE_LIBRARY'] = os.path.join(os.path.dirname(awx.__file__), 'plugins', 'isolated') - env['ANSIBLE_COLLECTIONS_PATHS'] = settings.AWX_ANSIBLE_COLLECTIONS_PATHS - set_pythonpath(os.path.join(settings.ANSIBLE_VENV_PATH, 'lib'), env) - - def finished_callback(runner_obj): - if runner_obj.status == 'failed' and runner_obj.config.playbook != 'check_isolated.yml': - # failed for clean_isolated.yml just means the playbook hasn't - # exited on the isolated host - stdout = runner_obj.stdout.read() - playbook_logger.error(stdout) - elif runner_obj.status == 'timeout': - # this means that the default idle timeout of - # (2 * AWX_ISOLATED_CONNECTION_TIMEOUT) was exceeded - # (meaning, we tried to sync with an isolated node, and we got - # no new output for 2 * AWX_ISOLATED_CONNECTION_TIMEOUT seconds) - # this _usually_ means SSH key auth from the controller -> - # isolated didn't work, and ssh is hung waiting on interactive - # input e.g., - # - # awx@isolated's password: - stdout = runner_obj.stdout.read() - playbook_logger.error(stdout) - else: - playbook_logger.info(runner_obj.stdout.read()) - - return { - 'project_dir': os.path.abspath(os.path.join(os.path.dirname(awx.__file__), 'playbooks')), - 'inventory': self.build_inventory(hosts), - 'envvars': env, - 'finished_callback': finished_callback, - 'verbosity': verbosity, - 'cancel_callback': self.canceled_callback, - 'settings': { - 'job_timeout': settings.AWX_ISOLATED_LAUNCH_TIMEOUT, - 'suppress_ansible_output': True, - }, - } - - def path_to(self, *args): - return os.path.join(self.private_data_dir, *args) - - def run_management_playbook(self, playbook, private_data_dir, idle_timeout=None, **kw): - iso_dir = tempfile.mkdtemp(prefix=playbook, dir=private_data_dir) - params = self.runner_params.copy() - params.get('envvars', dict())['ANSIBLE_CALLBACK_WHITELIST'] = 'profile_tasks' - params['playbook'] = playbook - params['private_data_dir'] = iso_dir - if idle_timeout: - params['settings']['idle_timeout'] = idle_timeout - else: - params['settings'].pop('idle_timeout', None) - params.update(**kw) - if all([getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True, getattr(settings, 'AWX_ISOLATED_PRIVATE_KEY', None)]): - params['ssh_key'] = settings.AWX_ISOLATED_PRIVATE_KEY - return ansible_runner.interface.run(**params) - - def dispatch(self, playbook=None, module=None, module_args=None): - """ - Ship the runner payload to a remote host for isolated execution. - """ - self.handled_events = set() - self.started_at = time.time() - - # exclude certain files from the rsync - rsync_exclude = [ - # don't rsync source control metadata (it can be huge!) - '- /project/.git', - '- /project/.svn', - # don't rsync job events that are in the process of being written - '- /artifacts/job_events/*-partial.json.tmp', - # don't rsync the ssh_key FIFO - '- /env/ssh_key', - # don't rsync kube config files - '- .kubeconfig*', - ] - - for filename, data in (['.rsync-filter', '\n'.join(rsync_exclude)],): - path = self.path_to(filename) - with open(path, 'w') as f: - f.write(data) - os.chmod(path, stat.S_IRUSR) - - extravars = { - 'src': self.private_data_dir, - 'dest': settings.AWX_ISOLATION_BASE_PATH, - 'ident': self.ident, - 'job_id': self.instance.id, - } - if playbook: - extravars['playbook'] = playbook - if module and module_args: - extravars['module'] = module - extravars['module_args'] = module_args - - logger.debug('Starting job {} on isolated host with `run_isolated.yml` playbook.'.format(self.instance.id)) - runner_obj = self.run_management_playbook( - 'run_isolated.yml', self.private_data_dir, idle_timeout=max(60, 2 * settings.AWX_ISOLATED_CONNECTION_TIMEOUT), extravars=extravars - ) - - if runner_obj.status == 'failed': - self.instance.result_traceback = runner_obj.stdout.read() - self.instance.save(update_fields=['result_traceback']) - return 'error', runner_obj.rc - - return runner_obj.status, runner_obj.rc - - def check(self, interval=None): - """ - Repeatedly poll the isolated node to determine if the job has run. - - On success, copy job artifacts to the controlling node. - On failure, continue to poll the isolated node (until the job timeout - is exceeded). - - For a completed job run, this function returns (status, rc), - representing the status and return code of the isolated - `ansible-playbook` run. - - :param interval: an interval (in seconds) to wait between status polls - """ - interval = interval if interval is not None else settings.AWX_ISOLATED_CHECK_INTERVAL - extravars = {'src': self.private_data_dir, 'job_id': self.instance.id} - status = 'failed' - rc = None - last_check = time.time() - - while status == 'failed': - canceled = self.canceled_callback() if self.canceled_callback else False - if not canceled and time.time() - last_check < interval: - # If the job isn't canceled, but we haven't waited `interval` seconds, wait longer - time.sleep(1) - continue - - if canceled: - logger.warning('Isolated job {} was manually canceled.'.format(self.instance.id)) - - logger.debug('Checking on isolated job {} with `check_isolated.yml`.'.format(self.instance.id)) - time_start = datetime.datetime.now() - runner_obj = self.run_management_playbook('check_isolated.yml', self.private_data_dir, extravars=extravars) - time_end = datetime.datetime.now() - time_diff = time_end - time_start - logger.debug('Finished checking on isolated job {} with `check_isolated.yml` took {} seconds.'.format(self.instance.id, time_diff.total_seconds())) - status, rc = runner_obj.status, runner_obj.rc - - if self.check_callback is not None and not self.captured_command_artifact: - command_path = self.path_to('artifacts', self.ident, 'command') - # If the configuration artifact has been synced back, update the model - if os.path.exists(command_path): - try: - with open(command_path, 'r') as f: - data = json.load(f) - self.check_callback(data) - self.captured_command_artifact = True - except json.decoder.JSONDecodeError: # Just in case it's not fully here yet. - pass - - self.consume_events() - - last_check = time.time() - - if status == 'successful': - status_path = self.path_to('artifacts', self.ident, 'status') - rc_path = self.path_to('artifacts', self.ident, 'rc') - if os.path.exists(status_path): - with open(status_path, 'r') as f: - status = f.readline() - with open(rc_path, 'r') as f: - rc = int(f.readline()) - else: - # if there's no status file, it means that runner _probably_ - # exited with a traceback (which should be logged to - # daemon.log) Record it so we can see how runner failed. - daemon_path = self.path_to('daemon.log') - if os.path.exists(daemon_path): - with open(daemon_path, 'r') as f: - self.instance.result_traceback = f.read() - self.instance.save(update_fields=['result_traceback']) - else: - logger.error('Failed to rsync daemon.log (is ansible-runner installed on the isolated host?)') - status = 'failed' - rc = 1 - - # consume events one last time just to be sure we didn't miss anything - # in the final sync - self.consume_events() - - return status, rc - - def consume_events(self): - # discover new events and ingest them - events_path = self.path_to('artifacts', self.ident, 'job_events') - - # it's possible that `events_path` doesn't exist *yet*, because runner - # hasn't actually written any events yet (if you ran e.g., a sleep 30) - # only attempt to consume events if any were rsynced back - if os.path.exists(events_path): - for event in set(os.listdir(events_path)) - self.handled_events: - path = os.path.join(events_path, event) - if os.path.exists(path) and os.path.isfile(path): - try: - event_data = json.load(open(os.path.join(events_path, event), 'r')) - except json.decoder.JSONDecodeError: - # This means the event we got back isn't valid JSON - # that can happen if runner is still partially - # writing an event file while it's rsyncing - # these event writes are _supposed_ to be atomic - # but it doesn't look like they actually are in - # practice - # in this scenario, just ignore this event and try it - # again on the next sync - continue - self.event_handler(event_data) - self.handled_events.add(event) - - def cleanup(self): - extravars = { - 'private_data_dir': self.private_data_dir, - 'cleanup_dirs': [ - self.private_data_dir, - ], - } - logger.debug('Cleaning up job {} on isolated host with `clean_isolated.yml` playbook.'.format(self.instance.id)) - self.run_management_playbook('clean_isolated.yml', self.private_data_dir, extravars=extravars) - - @classmethod - def update_capacity(cls, instance, task_result): - instance.version = 'ansible-runner-{}'.format(task_result['version']) - - if instance.capacity == 0 and task_result['capacity_cpu']: - logger.warning('Isolated instance {} has re-joined.'.format(instance.hostname)) - instance.cpu = int(task_result['cpu']) - instance.memory = int(task_result['mem']) - instance.cpu_capacity = int(task_result['capacity_cpu']) - instance.mem_capacity = int(task_result['capacity_mem']) - instance.capacity = get_system_task_capacity( - scale=instance.capacity_adjustment, cpu_capacity=int(task_result['capacity_cpu']), mem_capacity=int(task_result['capacity_mem']) - ) - instance.save(update_fields=['cpu', 'memory', 'cpu_capacity', 'mem_capacity', 'capacity', 'version', 'modified']) - - def health_check(self, instance_qs): - """ - :param instance_qs: List of Django objects representing the - isolated instances to manage - Runs playbook that will - - determine if instance is reachable - - find the instance capacity - - clean up orphaned private files - Performs save on each instance to update its capacity. - """ - instance_qs = [i for i in instance_qs if i.enabled] - if not len(instance_qs): - return - try: - private_data_dir = tempfile.mkdtemp(prefix='awx_iso_heartbeat_', dir=settings.AWX_ISOLATION_BASE_PATH) - self.runner_params = self.build_runner_params([instance.hostname for instance in instance_qs]) - self.runner_params['private_data_dir'] = private_data_dir - self.runner_params['forks'] = len(instance_qs) - runner_obj = self.run_management_playbook('heartbeat_isolated.yml', private_data_dir) - - for instance in instance_qs: - task_result = {} - try: - task_result = runner_obj.get_fact_cache(instance.hostname) - except Exception: - logger.exception('Failed to read status from isolated instances') - if 'awx_capacity_cpu' in task_result and 'awx_capacity_mem' in task_result: - task_result = { - 'cpu': task_result['awx_cpu'], - 'mem': task_result['awx_mem'], - 'capacity_cpu': task_result['awx_capacity_cpu'], - 'capacity_mem': task_result['awx_capacity_mem'], - 'version': task_result['awx_capacity_version'], - } - IsolatedManager.update_capacity(instance, task_result) - logger.debug('Isolated instance {} successful heartbeat'.format(instance.hostname)) - elif instance.capacity == 0: - logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format(instance.hostname)) - else: - logger.warning('Could not update status of isolated instance {}'.format(instance.hostname)) - if instance.is_lost(isolated=True): - instance.capacity = 0 - instance.save(update_fields=['capacity']) - logger.error('Isolated instance {} last checked in at {}, marked as lost.'.format(instance.hostname, instance.modified)) - finally: - if os.path.exists(private_data_dir): - shutil.rmtree(private_data_dir) - - def run(self, instance, private_data_dir, playbook, module, module_args, ident=None): - """ - Run a job on an isolated host. - - :param instance: a `model.Job` instance - :param private_data_dir: an absolute path on the local file system - where job-specific data should be written - (i.e., `/tmp/awx_N_xyz/`) - :param playbook: the playbook to run - :param module: the module to run - :param module_args: the module args to use - - For a completed job run, this function returns (status, rc), - representing the status and return code of the isolated - `ansible-playbook` run. - """ - self.ident = ident - self.instance = instance - self.private_data_dir = private_data_dir - self.runner_params = self.build_runner_params([instance.execution_node], verbosity=min(5, self.instance.verbosity)) - - status, rc = self.dispatch(playbook, module, module_args) - if status == 'successful': - status, rc = self.check() - return status, rc diff --git a/awx/main/management/commands/run_wsbroadcast.py b/awx/main/management/commands/run_wsbroadcast.py index 56a91f2a93..7ca47d77ff 100644 --- a/awx/main/management/commands/run_wsbroadcast.py +++ b/awx/main/management/commands/run_wsbroadcast.py @@ -10,7 +10,6 @@ from datetime import datetime as dt from django.core.management.base import BaseCommand from django.db import connection -from django.db.models import Q from django.db.migrations.executor import MigrationExecutor from awx.main.analytics.broadcast_websocket import ( @@ -140,8 +139,7 @@ class Command(BaseCommand): data[family.name] = family.samples[0].value me = Instance.objects.me() - # TODO: drop the isolated groups exclusion when the model is updated - hostnames = [i.hostname for i in Instance.objects.exclude(Q(hostname=me.hostname) | Q(rampart_groups__controller__isnull=False))] + hostnames = [i.hostname for i in Instance.objects.exclude(hostname=me.hostname)] host_stats = Command.get_connection_status(me, hostnames, data) lines = Command._format_lines(host_stats) diff --git a/awx/main/managers.py b/awx/main/managers.py index b3ff96783a..ada38ddd18 100644 --- a/awx/main/managers.py +++ b/awx/main/managers.py @@ -155,9 +155,6 @@ class InstanceManager(models.Manager): # NOTE: TODO: Likely to repurpose this once standalone ramparts are a thing return "tower" - def all_non_isolated(self): - return self.exclude(rampart_groups__controller__isnull=False) - class InstanceGroupManager(models.Manager): """A custom manager class for the Instance model. diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 063b5f47d7..a937ec6e59 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -33,7 +33,7 @@ import subprocess from django.conf import settings from django.db import transaction, DatabaseError, IntegrityError, ProgrammingError, connection from django.db.models.fields.related import ForeignKey -from django.utils.timezone import now, timedelta +from django.utils.timezone import now from django.utils.encoding import smart_str from django.contrib.auth.models import User from django.utils.translation import ugettext_lazy as _, gettext_noop @@ -84,7 +84,6 @@ from awx.main.models import ( from awx.main.constants import ACTIVE_STATES from awx.main.exceptions import AwxTaskError, PostRunError from awx.main.queue import CallbackQueueDispatcher -from awx.main.isolated import manager as isolated_manager from awx.main.dispatch.publish import task from awx.main.dispatch import get_local_queuename, reaper from awx.main.utils import ( @@ -170,8 +169,6 @@ def dispatch_startup(): # apply_cluster_membership_policies() cluster_node_heartbeat() - if Instance.objects.me().is_controller(): - awx_isolated_heartbeat() Metrics().clear_values() # Update Tower's rsyslog.conf file based on loggins settings in the db @@ -205,13 +202,8 @@ def apply_cluster_membership_policies(): started_compute = time.time() all_instances = list(Instance.objects.order_by('id')) all_groups = list(InstanceGroup.objects.prefetch_related('instances')) - iso_hostnames = set([]) - for ig in all_groups: - if ig.controller_id is not None: - iso_hostnames.update(ig.policy_instance_list) - considered_instances = [inst for inst in all_instances if inst.hostname not in iso_hostnames] - total_instances = len(considered_instances) + total_instances = len(all_instances) actual_groups = [] actual_instances = [] Group = namedtuple('Group', ['obj', 'instances', 'prior_instances']) @@ -232,18 +224,12 @@ def apply_cluster_membership_policies(): if group_actual.instances: logger.debug("Policy List, adding Instances {} to Group {}".format(group_actual.instances, ig.name)) - if ig.controller_id is None: - actual_groups.append(group_actual) - else: - # For isolated groups, _only_ apply the policy_instance_list - # do not add to in-memory list, so minimum rules not applied - logger.debug('Committing instances to isolated group {}'.format(ig.name)) - ig.instances.set(group_actual.instances) + actual_groups.append(group_actual) # Process Instance minimum policies next, since it represents a concrete lower bound to the # number of instances to make available to instance groups - actual_instances = [Node(obj=i, groups=[]) for i in considered_instances if i.managed_by_policy] - logger.debug("Total non-isolated instances:{} available for policy: {}".format(total_instances, len(actual_instances))) + actual_instances = [Node(obj=i, groups=[]) for i in all_instances if i.managed_by_policy] + logger.debug("Total instances: {}, available for policy: {}".format(total_instances, len(actual_instances))) for g in sorted(actual_groups, key=lambda x: len(x.instances)): policy_min_added = [] for i in sorted(actual_instances, key=lambda x: len(x.groups)): @@ -285,7 +271,7 @@ def apply_cluster_membership_policies(): logger.debug('Cluster policy no-op finished in {} seconds'.format(time.time() - started_compute)) return - # On a differential basis, apply instances to non-isolated groups + # On a differential basis, apply instances to groups with transaction.atomic(): for g in actual_groups: if g.obj.is_container_group: @@ -419,7 +405,7 @@ def cleanup_execution_environment_images(): def cluster_node_heartbeat(): logger.debug("Cluster node heartbeat task.") nowtime = now() - instance_list = list(Instance.objects.all_non_isolated()) + instance_list = list(Instance.objects.all()) this_inst = None lost_instances = [] @@ -503,30 +489,6 @@ def awx_k8s_reaper(): logger.exception("Failed to delete orphaned pod {} from {}".format(job.log_format, group)) -@task(queue=get_local_queuename) -def awx_isolated_heartbeat(): - local_hostname = settings.CLUSTER_HOST_ID - logger.debug("Controlling node checking for any isolated management tasks.") - poll_interval = settings.AWX_ISOLATED_PERIODIC_CHECK - # Get isolated instances not checked since poll interval - some buffer - nowtime = now() - accept_before = nowtime - timedelta(seconds=(poll_interval - 10)) - isolated_instance_qs = Instance.objects.filter( - rampart_groups__controller__instances__hostname=local_hostname, - ) - isolated_instance_qs = isolated_instance_qs.filter(last_isolated_check__lt=accept_before) | isolated_instance_qs.filter(last_isolated_check=None) - # Fast pass of isolated instances, claiming the nodes to update - with transaction.atomic(): - for isolated_instance in isolated_instance_qs: - isolated_instance.last_isolated_check = nowtime - # Prevent modified time from being changed, as in normal heartbeat - isolated_instance.save(update_fields=['last_isolated_check']) - # Slow pass looping over isolated IGs and their isolated instances - if len(isolated_instance_qs) > 0: - logger.debug("Managing isolated instances {}.".format(','.join([inst.hostname for inst in isolated_instance_qs]))) - isolated_manager.IsolatedManager(CallbackQueueDispatcher.dispatch).health_check(isolated_instance_qs) - - @task(queue=get_local_queuename) def awx_periodic_scheduler(): with advisory_lock('awx_periodic_scheduler_lock', wait=False) as acquired: @@ -1017,7 +979,7 @@ class BaseTask(object): else: env['PATH'] = os.path.join(settings.AWX_VENV_PATH, "bin") - def build_env(self, instance, private_data_dir, isolated, private_data_files=None): + def build_env(self, instance, private_data_dir, private_data_files=None): """ Build environment dictionary for ansible-playbook. """ @@ -1138,7 +1100,7 @@ class BaseTask(object): """ instance.log_lifecycle("post_run") - def final_run_hook(self, instance, status, private_data_dir, fact_modification_times, isolated_manager_instance=None): + def final_run_hook(self, instance, status, private_data_dir, fact_modification_times): """ Hook for any steps to run after job/task is marked as complete. """ @@ -1280,16 +1242,6 @@ class BaseTask(object): job_env[k] = v self.instance = self.update_model(self.instance.pk, job_args=json.dumps(runner_config.command), job_cwd=runner_config.cwd, job_env=job_env) - def check_handler(self, config): - """ - IsolatedManager callback triggered by the repeated checks of the isolated node - """ - job_env = build_safe_env(config['env']) - for k, v in self.safe_cred_env.items(): - if k in job_env: - job_env[k] = v - self.instance = self.update_model(self.instance.pk, job_args=json.dumps(config['command']), job_cwd=config['cwd'], job_env=job_env) - @with_path_cleanup def run(self, pk, **kwargs): """ @@ -1317,7 +1269,6 @@ class BaseTask(object): self.safe_env = {} self.safe_cred_env = {} private_data_dir = None - isolated_manager_instance = None # store a reference to the parent workflow job (if any) so we can include # it in event data JSON @@ -1325,7 +1276,6 @@ class BaseTask(object): self.parent_workflow_job_id = self.instance.get_workflow_job().id try: - isolated = self.instance.is_isolated() self.instance.send_notification_templates("running") private_data_dir = self.build_private_data_dir(self.instance) self.pre_run_hook(self.instance, private_data_dir) @@ -1359,7 +1309,7 @@ class BaseTask(object): passwords = self.build_passwords(self.instance, kwargs) self.build_extra_vars_file(self.instance, private_data_dir) args = self.build_args(self.instance, private_data_dir, passwords) - env = self.build_env(self.instance, private_data_dir, isolated, private_data_files=private_data_files) + env = self.build_env(self.instance, private_data_dir, private_data_files=private_data_files) self.safe_env = build_safe_env(env) credentials = self.build_credentials_list(self.instance) @@ -1460,7 +1410,7 @@ class BaseTask(object): self.instance = self.update_model(pk, status=status, emitted_events=self.event_ct, **extra_update_fields) try: - self.final_run_hook(self.instance, status, private_data_dir, fact_modification_times, isolated_manager_instance=isolated_manager_instance) + self.final_run_hook(self.instance, status, private_data_dir, fact_modification_times) except Exception: logger.exception('{} Final run hook errored.'.format(self.instance.log_format)) @@ -1545,11 +1495,11 @@ class RunJob(BaseTask): return passwords - def build_env(self, job, private_data_dir, isolated=False, private_data_files=None): + def build_env(self, job, private_data_dir, private_data_files=None): """ Build environment dictionary for ansible-playbook. """ - env = super(RunJob, self).build_env(job, private_data_dir, isolated=isolated, private_data_files=private_data_files) + env = super(RunJob, self).build_env(job, private_data_dir, private_data_files=private_data_files) if private_data_files is None: private_data_files = {} # Set environment variables needed for inventory and job event @@ -1560,10 +1510,9 @@ class RunJob(BaseTask): env['PROJECT_REVISION'] = job.project.scm_revision env['ANSIBLE_RETRY_FILES_ENABLED'] = "False" env['MAX_EVENT_RES'] = str(settings.MAX_EVENT_RES_DATA) - if not isolated: - if hasattr(settings, 'AWX_ANSIBLE_CALLBACK_PLUGINS') and settings.AWX_ANSIBLE_CALLBACK_PLUGINS: - env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(settings.AWX_ANSIBLE_CALLBACK_PLUGINS) - env['AWX_HOST'] = settings.TOWER_URL_BASE + if hasattr(settings, 'AWX_ANSIBLE_CALLBACK_PLUGINS') and settings.AWX_ANSIBLE_CALLBACK_PLUGINS: + env['ANSIBLE_CALLBACK_PLUGINS'] = ':'.join(settings.AWX_ANSIBLE_CALLBACK_PLUGINS) + env['AWX_HOST'] = settings.TOWER_URL_BASE # Create a directory for ControlPath sockets that is unique to each job cp_dir = os.path.join(private_data_dir, 'cp') @@ -1794,9 +1743,6 @@ class RunJob(BaseTask): if sync_needs: pu_ig = job.instance_group pu_en = job.execution_node - if job.is_isolated() is True: - pu_ig = pu_ig.controller - pu_en = settings.CLUSTER_HOST_ID sync_metafields = dict( launch_type="sync", @@ -1851,7 +1797,7 @@ class RunJob(BaseTask): # ran inside of the event saving code update_smart_memberships_for_inventory(job.inventory) - def final_run_hook(self, job, status, private_data_dir, fact_modification_times, isolated_manager_instance=None): + def final_run_hook(self, job, status, private_data_dir, fact_modification_times): super(RunJob, self).final_run_hook(job, status, private_data_dir, fact_modification_times) if not private_data_dir: # If there's no private data dir, that means we didn't get into the @@ -1863,8 +1809,6 @@ class RunJob(BaseTask): os.path.join(private_data_dir, 'artifacts', 'fact_cache'), fact_modification_times, ) - if isolated_manager_instance and not job.is_container_group_task: - isolated_manager_instance.cleanup() try: inventory = job.inventory @@ -1928,11 +1872,11 @@ class RunProjectUpdate(BaseTask): passwords['scm_password'] = project_update.credential.get_input('password', default='') return passwords - def build_env(self, project_update, private_data_dir, isolated=False, private_data_files=None): + def build_env(self, project_update, private_data_dir, private_data_files=None): """ Build environment dictionary for ansible-playbook. """ - env = super(RunProjectUpdate, self).build_env(project_update, private_data_dir, isolated=isolated, private_data_files=private_data_files) + env = super(RunProjectUpdate, self).build_env(project_update, private_data_dir, private_data_files=private_data_files) env['ANSIBLE_RETRY_FILES_ENABLED'] = str(False) env['ANSIBLE_ASK_PASS'] = str(False) env['ANSIBLE_BECOME_ASK_PASS'] = str(False) @@ -2387,14 +2331,14 @@ class RunInventoryUpdate(BaseTask): injector = InventorySource.injectors[inventory_update.source]() return injector.build_private_data(inventory_update, private_data_dir) - def build_env(self, inventory_update, private_data_dir, isolated, private_data_files=None): + def build_env(self, inventory_update, private_data_dir, private_data_files=None): """Build environment dictionary for ansible-inventory. Most environment variables related to credentials or configuration are accomplished by the inventory source injectors (in this method) or custom credential type injectors (in main run method). """ - env = super(RunInventoryUpdate, self).build_env(inventory_update, private_data_dir, isolated, private_data_files=private_data_files) + env = super(RunInventoryUpdate, self).build_env(inventory_update, private_data_dir, private_data_files=private_data_files) if private_data_files is None: private_data_files = {} @@ -2711,11 +2655,11 @@ class RunAdHocCommand(BaseTask): passwords[field] = value return passwords - def build_env(self, ad_hoc_command, private_data_dir, isolated=False, private_data_files=None): + def build_env(self, ad_hoc_command, private_data_dir, private_data_files=None): """ Build environment dictionary for ansible. """ - env = super(RunAdHocCommand, self).build_env(ad_hoc_command, private_data_dir, isolated=isolated, private_data_files=private_data_files) + env = super(RunAdHocCommand, self).build_env(ad_hoc_command, private_data_dir, private_data_files=private_data_files) # Set environment variables needed for inventory and ad hoc event # callbacks to work. env['AD_HOC_COMMAND_ID'] = str(ad_hoc_command.pk) @@ -2821,11 +2765,6 @@ class RunAdHocCommand(BaseTask): d[r'Password:\s*?$'] = 'ssh_password' return d - def final_run_hook(self, adhoc_job, status, private_data_dir, fact_modification_times, isolated_manager_instance=None): - super(RunAdHocCommand, self).final_run_hook(adhoc_job, status, private_data_dir, fact_modification_times) - if isolated_manager_instance: - isolated_manager_instance.cleanup() - @task(queue=get_local_queuename) class RunSystemJob(BaseTask): @@ -2867,8 +2806,8 @@ class RunSystemJob(BaseTask): os.chmod(path, stat.S_IRUSR) return path - def build_env(self, instance, private_data_dir, isolated=False, private_data_files=None): - base_env = super(RunSystemJob, self).build_env(instance, private_data_dir, isolated=isolated, private_data_files=private_data_files) + def build_env(self, instance, private_data_dir, private_data_files=None): + base_env = super(RunSystemJob, self).build_env(instance, private_data_dir, private_data_files=private_data_files) # TODO: this is able to run by turning off isolation # the goal is to run it a container instead env = dict(os.environ.items()) diff --git a/awx/main/tests/unit/test_tasks.py b/awx/main/tests/unit/test_tasks.py index e6460dbc07..1cd7dd9069 100644 --- a/awx/main/tests/unit/test_tasks.py +++ b/awx/main/tests/unit/test_tasks.py @@ -645,102 +645,6 @@ class TestAdhocRun(TestJobExecution): assert extra_vars['awx_user_name'] == "angry-spud" -@pytest.mark.skip(reason="Isolated code path needs updating after runner integration") -class TestIsolatedExecution(TestJobExecution): - - ISOLATED_HOST = 'some-isolated-host' - ISOLATED_CONTROLLER_HOST = 'some-isolated-controller-host' - - @pytest.fixture - def job(self): - job = Job(pk=1, id=1, project=Project(), inventory=Inventory(), job_template=JobTemplate(id=1, name='foo')) - job.controller_node = self.ISOLATED_CONTROLLER_HOST - job.execution_node = self.ISOLATED_HOST - return job - - def test_with_ssh_credentials(self, job): - ssh = CredentialType.defaults['ssh']() - credential = Credential(pk=1, credential_type=ssh, inputs={'username': 'bob', 'password': 'secret', 'ssh_key_data': self.EXAMPLE_PRIVATE_KEY}) - credential.inputs['password'] = encrypt_field(credential, 'password') - job.credentials.add(credential) - - private_data = tempfile.mkdtemp(prefix='awx_') - self.task.build_private_data_dir = mock.Mock(return_value=private_data) - - def _mock_job_artifacts(*args, **kw): - artifacts = os.path.join(private_data, 'artifacts') - if not os.path.exists(artifacts): - os.makedirs(artifacts) - if 'run_isolated.yml' in args[0]: - for filename, data in ( - ['status', 'successful'], - ['rc', '0'], - ['stdout', 'IT WORKED!'], - ): - with open(os.path.join(artifacts, filename), 'w') as f: - f.write(data) - return ('successful', 0) - - self.run_pexpect.side_effect = _mock_job_artifacts - self.task.run(self.pk) - - playbook_run = self.run_pexpect.call_args_list[0][0] - assert ' '.join(playbook_run[0]).startswith( - ' '.join( - [ - 'ansible-playbook', - 'run_isolated.yml', - '-u', - settings.AWX_ISOLATED_USERNAME, - '-T', - str(settings.AWX_ISOLATED_CONNECTION_TIMEOUT), - '-i', - self.ISOLATED_HOST + ',', - '-e', - ] - ) - ) - extra_vars = playbook_run[0][playbook_run[0].index('-e') + 1] - extra_vars = json.loads(extra_vars) - assert extra_vars['dest'] == '/tmp' - assert extra_vars['src'] == private_data - - def test_systemctl_failure(self): - # If systemctl fails, read the contents of `artifacts/systemctl_logs` - mock_get = mock.Mock() - ssh = CredentialType.defaults['ssh']() - credential = Credential( - pk=1, - credential_type=ssh, - inputs={ - 'username': 'bob', - }, - ) - self.instance.credentials.add(credential) - - private_data = tempfile.mkdtemp(prefix='awx_') - self.task.build_private_data_dir = mock.Mock(return_value=private_data) - inventory = json.dumps({"all": {"hosts": ["localhost"]}}) - - def _mock_job_artifacts(*args, **kw): - artifacts = os.path.join(private_data, 'artifacts') - if not os.path.exists(artifacts): - os.makedirs(artifacts) - if 'run_isolated.yml' in args[0]: - for filename, data in (['daemon.log', 'ERROR IN RUN.PY'],): - with open(os.path.join(artifacts, filename), 'w') as f: - f.write(data) - return ('successful', 0) - - self.run_pexpect.side_effect = _mock_job_artifacts - - with mock.patch('time.sleep'): - with mock.patch('requests.get') as mock_get: - mock_get.return_value = mock.Mock(content=inventory) - with pytest.raises(Exception): - self.task.run(self.pk, self.ISOLATED_HOST) - - class TestJobCredentials(TestJobExecution): @pytest.fixture def job(self, execution_environment): @@ -1625,7 +1529,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) assert 'AWS_ACCESS_KEY_ID' not in env assert 'AWS_SECRET_ACCESS_KEY' not in env @@ -1645,7 +1549,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = build_safe_env(env) @@ -1669,7 +1573,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = {} credentials = task.build_credentials_list(inventory_update) @@ -1706,7 +1610,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = build_safe_env(env) @@ -1736,7 +1640,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = build_safe_env(env) @@ -1763,7 +1667,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): def run(expected_gce_zone): private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = {} credentials = task.build_credentials_list(inventory_update) for credential in credentials: @@ -1797,7 +1701,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) path = os.path.join(private_data_dir, os.path.basename(env['OS_CLIENT_CONFIG_FILE'])) shade_config = open(path, 'r').read() @@ -1832,7 +1736,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) safe_env = build_safe_env(env) assert env["FOREMAN_SERVER"] == "https://example.org" @@ -1856,7 +1760,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_cloud_credential = get_cred inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) - env = task.build_env(inventory_update, private_data_dir, False) + env = task.build_env(inventory_update, private_data_dir) safe_env = build_safe_env(env) @@ -1888,7 +1792,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): inventory_update.get_cloud_credential = get_cred inventory_update.get_extra_credentials = mocker.Mock(return_value=[]) - env = task.build_env(inventory_update, private_data_dir, False) + env = task.build_env(inventory_update, private_data_dir) safe_env = {} credentials = task.build_credentials_list(inventory_update) for credential in credentials: @@ -1919,7 +1823,7 @@ class TestInventoryUpdateCredentials(TestJobExecution): settings.AWX_TASK_ENV = {'FOO': 'BAR'} private_data_files = task.build_private_data_files(inventory_update, private_data_dir) - env = task.build_env(inventory_update, private_data_dir, False, private_data_files) + env = task.build_env(inventory_update, private_data_dir, private_data_files) assert env['FOO'] == 'BAR' diff --git a/awx/main/wsbroadcast.py b/awx/main/wsbroadcast.py index 7e0ca3a7e2..b35747aee3 100644 --- a/awx/main/wsbroadcast.py +++ b/awx/main/wsbroadcast.py @@ -32,14 +32,7 @@ def unwrap_broadcast_msg(payload: dict): def get_broadcast_hosts(): Instance = apps.get_model('main', 'Instance') - instances = ( - # TODO: no longer filter for non-isolated after the models change - Instance.objects.filter(rampart_groups__controller__isnull=True) - .exclude(hostname=Instance.objects.me().hostname) - .order_by('hostname') - .values('hostname', 'ip_address') - .distinct() - ) + instances = Instance.objects.exclude(hostname=Instance.objects.me().hostname).order_by('hostname').values('hostname', 'ip_address').distinct() return {i['hostname']: i['ip_address'] or i['hostname'] for i in instances} diff --git a/awx/playbooks/check_isolated.yml b/awx/playbooks/check_isolated.yml deleted file mode 100644 index 472b772fbb..0000000000 --- a/awx/playbooks/check_isolated.yml +++ /dev/null @@ -1,70 +0,0 @@ ---- -# The following variables will be set by the runner of this playbook: -# src: /tmp/some/path/private_data_dir/ - -- name: Poll for status of active job. - hosts: all - gather_facts: false - collections: - - ansible.posix - - tasks: - - name: "Output job the playbook is running for" - debug: - msg: "Checking on job {{ job_id }}" - - - name: Determine if daemon process is alive. - shell: "ansible-runner is-alive {{src}}" - register: is_alive - ignore_errors: true - - - name: Copy artifacts from the isolated host. - synchronize: - src: "{{src}}/artifacts/" - dest: "{{src}}/artifacts/" - mode: pull - delete: true - recursive: true - when: ansible_kubectl_config is not defined - - - name: Copy daemon log from the isolated host - synchronize: - src: "{{src}}/daemon.log" - dest: "{{src}}/daemon.log" - mode: pull - when: ansible_kubectl_config is not defined - - - name: Copy artifacts from pod - synchronize: - src: "{{src}}/artifacts/" - dest: "{{src}}/artifacts/" - mode: pull - delete: true - recursive: true - set_remote_user: false - rsync_opts: - - "--blocking-io" - - "--rsh=$RSH" - environment: - RSH: "oc rsh --config={{ ansible_kubectl_config }}" - delegate_to: localhost - when: ansible_kubectl_config is defined - - - name: Copy daemon log from pod - synchronize: - src: "{{src}}/daemon.log" - dest: "{{src}}/daemon.log" - mode: pull - set_remote_user: false - rsync_opts: - - "--blocking-io" - - "--rsh=$RSH" - environment: - RSH: "oc rsh --config={{ ansible_kubectl_config }}" - delegate_to: localhost - when: ansible_kubectl_config is defined - - - name: Fail if previous check determined that process is not alive. - fail: - msg: "isolated task is still running" - when: "is_alive.rc == 0" diff --git a/awx/playbooks/clean_isolated.yml b/awx/playbooks/clean_isolated.yml deleted file mode 100644 index 63c044b8a1..0000000000 --- a/awx/playbooks/clean_isolated.yml +++ /dev/null @@ -1,31 +0,0 @@ ---- - -# The following variables will be set by the runner of this playbook: -# cleanup_dirs: ['/tmp/path/private_data_dir/', '/tmp//path/proot_temp_dir/'] -# private_data_dir: '/tmp/path/private_data_dir/' - -- name: Clean up from isolated job run. - hosts: all - gather_facts: false - - tasks: - - - name: cancel the job - command: "ansible-runner stop {{private_data_dir}}" - ignore_errors: true - - - name: remove build artifacts - file: - path: '{{item}}' - state: absent - register: result - with_items: "{{cleanup_dirs}}" - until: result is succeeded - ignore_errors: true - retries: 3 - delay: 5 - - - name: fail if build artifacts were not cleaned - fail: - msg: 'Unable to cleanup build artifacts' - when: not result is succeeded diff --git a/awx/playbooks/heartbeat_isolated.yml b/awx/playbooks/heartbeat_isolated.yml deleted file mode 100644 index 7963d5fbe2..0000000000 --- a/awx/playbooks/heartbeat_isolated.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -- name: Periodic background status check of isolated instances. - hosts: all - gather_facts: false - - tasks: - - - name: Get capacity of the instance - awx_capacity: - - - name: Remove any stale temporary files - awx_isolated_cleanup: diff --git a/awx/playbooks/run_isolated.yml b/awx/playbooks/run_isolated.yml deleted file mode 100644 index 76ea42d17c..0000000000 --- a/awx/playbooks/run_isolated.yml +++ /dev/null @@ -1,61 +0,0 @@ ---- - -# The following variables will be set by the runner of this playbook: -# src: /tmp/some/path/private_data_dir -# dest: /tmp/some/path/ - -- name: Prepare data, dispatch job in isolated environment. - hosts: all - gather_facts: false - vars: - secret: "{{ lookup('pipe', 'cat ' + src + '/env/ssh_key') }}" - collections: - - ansible.posix - - tasks: - - name: "Output job the playbook is running for" - debug: - msg: "Checking on job {{ job_id }}" - - - name: synchronize job environment with isolated host - synchronize: - copy_links: true - src: "{{ src }}" - dest: "{{ dest }}" - when: ansible_kubectl_config is not defined - - - name: synchronize job environment with remote job container - synchronize: - copy_links: true - src: "{{ src }}" - dest: "{{ dest }}" - set_remote_user: false - rsync_opts: - - "--blocking-io" - - "--rsh=$RSH" - environment: - RSH: "oc rsh --config={{ ansible_kubectl_config }}" - delegate_to: localhost - when: ansible_kubectl_config is defined - - - local_action: stat path="{{src}}/env/ssh_key" - register: key - - - name: create a named pipe for secret environment data - command: "mkfifo {{src}}/env/ssh_key" - when: key.stat.exists - - - name: spawn the playbook - command: "ansible-runner start {{src}} -p '{{playbook}}' -i {{ident}}" - when: playbook is defined - - - name: spawn the adhoc command - command: "ansible-runner start {{src}} -m {{module}} -a {{module_args}} -i {{ident}}" - when: module is defined - - - name: write the secret environment data - mkfifo: - content: "{{secret}}" - path: "{{src}}/env/ssh_key" - when: key.stat.exists - no_log: true diff --git a/awx/plugins/isolated/awx_capacity.py b/awx/plugins/isolated/awx_capacity.py deleted file mode 100644 index 2f33a8ffad..0000000000 --- a/awx/plugins/isolated/awx_capacity.py +++ /dev/null @@ -1,73 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from ansible.module_utils._text import to_text -from ansible.module_utils.basic import AnsibleModule - -import subprocess -import os -import psutil - - -def get_cpu_capacity(): - env_forkcpu = os.getenv('SYSTEM_TASK_FORKS_CPU', None) - cpu = psutil.cpu_count() - - if env_forkcpu: - forkcpu = int(env_forkcpu) - else: - forkcpu = 4 - return (cpu, cpu * forkcpu) - - -def get_mem_capacity(): - env_forkmem = os.getenv('SYSTEM_TASK_FORKS_MEM', None) - if env_forkmem: - forkmem = int(env_forkmem) - else: - forkmem = 100 - - mem = psutil.virtual_memory().total - return (mem, max(1, ((mem / 1024 / 1024) - 2048) / forkmem)) - - -def main(): - module = AnsibleModule(argument_spec=dict()) - - ar = module.get_bin_path('ansible-runner', required=True) - - try: - version = subprocess.check_output([ar, '--version'], stderr=subprocess.STDOUT).strip() - except subprocess.CalledProcessError as e: - module.fail_json(msg=to_text(e)) - return - # NOTE: Duplicated with awx.main.utils.common capacity utilities - cpu, capacity_cpu = get_cpu_capacity() - mem, capacity_mem = get_mem_capacity() - - # Module never results in a change - module.exit_json( - changed=False, - capacity_cpu=capacity_cpu, - capacity_mem=capacity_mem, - version=version, - ansible_facts=dict(awx_cpu=cpu, awx_mem=mem, awx_capacity_cpu=capacity_cpu, awx_capacity_mem=capacity_mem, awx_capacity_version=version), - ) - - -if __name__ == '__main__': - main() diff --git a/awx/plugins/isolated/awx_isolated_cleanup.py b/awx/plugins/isolated/awx_isolated_cleanup.py deleted file mode 100644 index 7f58a1f74a..0000000000 --- a/awx/plugins/isolated/awx_isolated_cleanup.py +++ /dev/null @@ -1,68 +0,0 @@ -# Copyright (c) 2017 Ansible by Red Hat -# -# This file is part of Ansible Tower, but depends on code imported from Ansible. -# -# Ansible is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# Ansible is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with Ansible. If not, see . - -from ansible.module_utils.basic import AnsibleModule - -import glob -import os -import re -import shutil -import datetime -import subprocess - - -def main(): - module = AnsibleModule(argument_spec=dict()) - changed = False - paths_removed = set([]) - - # If a folder was last modified before this datetime, it will always be deleted - folder_cutoff = datetime.datetime.now() - datetime.timedelta(days=7) - # If a folder does not have an associated job running and is older than - # this datetime, then it will be deleted because its job has finished - job_cutoff = datetime.datetime.now() - datetime.timedelta(hours=1) - - for search_pattern in ['/tmp/awx_[0-9]*_*', '/tmp/ansible_runner_pi_*']: - for path in glob.iglob(search_pattern): - st = os.stat(path) - modtime = datetime.datetime.fromtimestamp(st.st_mtime) - - if modtime > job_cutoff: - continue - elif modtime > folder_cutoff: - try: - re_match = re.match(r'\/tmp\/awx_\d+_.+', path) - if re_match is not None: - try: - if subprocess.check_call(['ansible-runner', 'is-alive', path]) == 0: - continue - except subprocess.CalledProcessError: - # the job isn't running anymore, clean up this path - module.debug('Deleting path {} its job has completed.'.format(path)) - except (ValueError, IndexError): - continue - else: - module.debug('Deleting path {} because modification date is too old.'.format(path)) - changed = True - paths_removed.add(path) - shutil.rmtree(path) - - module.exit_json(changed=changed, paths_removed=list(paths_removed)) - - -if __name__ == '__main__': - main() diff --git a/awx/plugins/isolated/mkfifo.py b/awx/plugins/isolated/mkfifo.py deleted file mode 100755 index 45741c2ad3..0000000000 --- a/awx/plugins/isolated/mkfifo.py +++ /dev/null @@ -1,31 +0,0 @@ -import os -import stat - -from ansible.module_utils.basic import AnsibleModule - - -# -# the purpose of this plugin is to call mkfifo and -# write raw SSH key data into the fifo created on the remote isolated host -# - - -def main(): - module = AnsibleModule(argument_spec={'path': {'required': True, 'type': 'str'}, 'content': {'required': True, 'type': 'str'}}, supports_check_mode=False) - - path = module.params['path'] - os.chmod(path, stat.S_IRUSR | stat.S_IWUSR) - with open(path, 'w') as fifo: - data = module.params['content'] - if 'OPENSSH PRIVATE KEY' in data and not data.endswith('\n'): - # we use ansible's lookup() to read this file from the disk, - # but ansible's lookup() *strips* newlines - # OpenSSH wants certain private keys to end with a newline (or it - # won't accept them) - data += '\n' - fifo.write(data) - module.exit_json(dest=path, changed=True) - - -if __name__ == '__main__': - main() diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index aa6e0ae4a0..4c758b5ca1 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -126,7 +126,7 @@ LOGIN_URL = '/api/login/' PROJECTS_ROOT = '/var/lib/awx/projects/' # Absolute filesystem path to the directory to host collections for -# running inventory imports, isolated playbooks +# running inventory imports AWX_ANSIBLE_COLLECTIONS_PATHS = os.path.join(BASE_DIR, 'vendor', 'awx_ansible_collections') # Absolute filesystem path to the directory for job status stdout (default for @@ -440,7 +440,6 @@ CELERYBEAT_SCHEDULE = { 'k8s_reaper': {'task': 'awx.main.tasks.awx_k8s_reaper', 'schedule': timedelta(seconds=60), 'options': {'expires': 50}}, 'send_subsystem_metrics': {'task': 'awx.main.analytics.analytics_tasks.send_subsystem_metrics', 'schedule': timedelta(seconds=20)}, 'cleanup_images': {'task': 'awx.main.tasks.cleanup_execution_environment_images', 'schedule': timedelta(hours=3)}, - # 'isolated_heartbeat': set up at the end of production.py and development.py } # Django Caching Configuration @@ -854,12 +853,6 @@ LOGGING = { 'filename': os.path.join(LOG_ROOT, 'tower_rbac_migrations.log'), 'formatter': 'simple', }, - 'isolated_manager': { - 'level': 'WARNING', - 'class': 'logging.handlers.WatchedFileHandler', - 'filename': os.path.join(LOG_ROOT, 'isolated_manager.log'), - 'formatter': 'simple', - }, 'job_lifecycle': { 'level': 'DEBUG', 'class': 'logging.handlers.WatchedFileHandler', @@ -881,8 +874,6 @@ LOGGING = { 'awx.main.dispatch': {'handlers': ['dispatcher']}, 'awx.main.consumers': {'handlers': ['console', 'file', 'tower_warnings'], 'level': 'INFO'}, 'awx.main.wsbroadcast': {'handlers': ['wsbroadcast']}, - 'awx.isolated.manager': {'level': 'WARNING', 'handlers': ['console', 'file', 'isolated_manager'], 'propagate': True}, - 'awx.isolated.manager.playbooks': {'handlers': ['management_playbooks'], 'propagate': False}, 'awx.main.commands.inventory_import': {'handlers': ['inventory_import'], 'propagate': False}, 'awx.main.tasks': {'handlers': ['task_system', 'external_logger'], 'propagate': False}, 'awx.main.analytics': {'handlers': ['task_system', 'external_logger'], 'level': 'INFO', 'propagate': False}, diff --git a/awx/settings/development.py b/awx/settings/development.py index 66dc12f50f..e836a723f6 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -35,9 +35,6 @@ LOGGING['handlers']['console']['()'] = 'awx.main.utils.handlers.ColorHandler' # LOGGING['handlers']['task_system'] = LOGGING['handlers']['console'].copy() # noqa COLOR_LOGS = True -# Pipe management playbook output to console -LOGGING['loggers']['awx.isolated.manager.playbooks']['propagate'] = True # noqa - # celery is annoyingly loud when docker containers start LOGGING['loggers'].pop('celery', None) # noqa # avoid awx.main.dispatch WARNING-level scaling worker up/down messages @@ -136,17 +133,6 @@ if "pytest" in sys.modules: } } - -CELERYBEAT_SCHEDULE.update( - { # noqa - 'isolated_heartbeat': { - 'task': 'awx.main.tasks.awx_isolated_heartbeat', - 'schedule': timedelta(seconds=AWX_ISOLATED_PERIODIC_CHECK), # noqa - 'options': {'expires': AWX_ISOLATED_PERIODIC_CHECK * 2}, # noqa - } - } -) - CLUSTER_HOST_ID = socket.gethostname() AWX_CALLBACK_PROFILE = True diff --git a/awx/settings/production.py b/awx/settings/production.py index f5a1bd7a7f..c6511cb5b1 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -91,14 +91,4 @@ except IOError: # The below runs AFTER all of the custom settings are imported. -CELERYBEAT_SCHEDULE.update( - { # noqa - 'isolated_heartbeat': { - 'task': 'awx.main.tasks.awx_isolated_heartbeat', - 'schedule': timedelta(seconds=AWX_ISOLATED_PERIODIC_CHECK), # noqa - 'options': {'expires': AWX_ISOLATED_PERIODIC_CHECK * 2}, # noqa - } - } -) - DATABASES['default'].setdefault('OPTIONS', dict()).setdefault('application_name', f'{CLUSTER_HOST_ID}-{os.getpid()}-{" ".join(sys.argv)}'[:63]) # noqa diff --git a/tools/ansible/roles/dockerfile/files/settings.py b/tools/ansible/roles/dockerfile/files/settings.py index c2abeb2df2..e2bbd8cb5b 100644 --- a/tools/ansible/roles/dockerfile/files/settings.py +++ b/tools/ansible/roles/dockerfile/files/settings.py @@ -58,7 +58,6 @@ LOGGING['loggers']['django_auth_ldap']['handlers'] = ['console'] LOGGING['loggers']['social']['handlers'] = ['console'] LOGGING['loggers']['system_tracking_migrations']['handlers'] = ['console'] LOGGING['loggers']['rbac_migrations']['handlers'] = ['console'] -LOGGING['loggers']['awx.isolated.manager.playbooks']['handlers'] = ['console'] LOGGING['handlers']['callback_receiver'] = {'class': 'logging.NullHandler'} LOGGING['handlers']['task_system'] = {'class': 'logging.NullHandler'} LOGGING['handlers']['tower_warnings'] = {'class': 'logging.NullHandler'}