From 84eacfc3608d06055e98757f2882586ff795768a Mon Sep 17 00:00:00 2001 From: Ryan Petrello Date: Wed, 13 Jun 2018 08:51:42 -0400 Subject: [PATCH] fix a few isolated dev issues the main goal of this change is to make `make docker-isolated` work out of the box - specify the proper version for awx-expect --version - update some deprecated playbook bits - change the isolated container to privileged so bwrap will work - fix awx-manage test_isolated_connection - expedite the first isolated heartbeat so you don't have to wait 10m; this is accomplished by _not_ setting Instance.last_isolated_check to now() at insertion time (which causes the next check not to happen for 10 minutes) - fix a bug that caused isolated node execution to fail when bwrap was enabled see: https://github.com/ansible/tower/issues/2150 This reverts commit 9863fe71dcaa987ed28d70ac6873ece058344ab3. --- Makefile | 4 ++-- awx/main/expect/isolated_manager.py | 1 + .../management/commands/test_isolated_connection.py | 10 ++++------ awx/main/migrations/0006_v320_release.py | 2 +- awx/main/models/ha.py | 1 - awx/main/tasks.py | 6 ++++++ awx/playbooks/clean_isolated.yml | 4 ++-- tools/docker-isolated-override.yml | 4 +--- tools/docker-isolated/Dockerfile | 3 --- 9 files changed, 17 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 0b9cc2a61c..658f135d6e 100644 --- a/Makefile +++ b/Makefile @@ -219,7 +219,7 @@ init: if [ "$(AWX_GROUP_QUEUES)" == "tower,thepentagon" ]; then \ $(MANAGEMENT_COMMAND) provision_instance --hostname=isolated; \ $(MANAGEMENT_COMMAND) register_queue --queuename='thepentagon' --hostnames=isolated --controller=tower; \ - $(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat > /root/.ssh/authorized_keys'; \ + $(MANAGEMENT_COMMAND) generate_isolated_key | ssh -o "StrictHostKeyChecking no" root@isolated 'cat >> /root/.ssh/authorized_keys'; \ fi; # Refresh development environment after pulling new code. @@ -560,7 +560,7 @@ docker-isolated: TAG=$(COMPOSE_TAG) DEV_DOCKER_TAG_BASE=$(DEV_DOCKER_TAG_BASE) docker-compose -f tools/docker-compose.yml -f tools/docker-isolated-override.yml create docker start tools_awx_1 docker start tools_isolated_1 - echo "__version__ = '`python setup.py --version`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python2.7/site-packages/awx.py" + echo "__version__ = '`git describe --long | cut -d - -f 1-1`'" | docker exec -i tools_isolated_1 /bin/bash -c "cat > /venv/awx/lib/python2.7/site-packages/awx.py" if [ "`docker exec -i -t tools_isolated_1 cat /root/.ssh/authorized_keys`" == "`docker exec -t tools_awx_1 cat /root/.ssh/id_rsa.pub`" ]; then \ echo "SSH keys already copied to isolated instance"; \ else \ diff --git a/awx/main/expect/isolated_manager.py b/awx/main/expect/isolated_manager.py index 9b4ce1db6a..a723d13fa2 100644 --- a/awx/main/expect/isolated_manager.py +++ b/awx/main/expect/isolated_manager.py @@ -434,6 +434,7 @@ class IsolatedManager(object): task_result = {} if 'capacity_cpu' in task_result and 'capacity_mem' in task_result: cls.update_capacity(instance, task_result, awx_application_version) + logger.debug('Isolated instance {} successful heartbeat'.format(instance.hostname)) elif instance.capacity == 0: logger.debug('Isolated instance {} previously marked as lost, could not re-join.'.format( instance.hostname)) diff --git a/awx/main/management/commands/test_isolated_connection.py b/awx/main/management/commands/test_isolated_connection.py index 23bd20bf50..efaf881535 100644 --- a/awx/main/management/commands/test_isolated_connection.py +++ b/awx/main/management/commands/test_isolated_connection.py @@ -3,7 +3,6 @@ import shutil import subprocess import sys import tempfile -from optparse import make_option from django.conf import settings from django.core.management.base import BaseCommand, CommandError @@ -15,10 +14,9 @@ class Command(BaseCommand): """Tests SSH connectivity between a controller and target isolated node""" help = 'Tests SSH connectivity between a controller and target isolated node' - option_list = BaseCommand.option_list + ( - make_option('--hostname', dest='hostname', type='string', - help='Hostname of an isolated node'), - ) + def add_arguments(self, parser): + parser.add_argument('--hostname', dest='hostname', type=str, + help='Hostname of an isolated node') def handle(self, *args, **options): hostname = options.get('hostname') @@ -30,7 +28,7 @@ class Command(BaseCommand): args = [ 'ansible', 'all', '-i', '{},'.format(hostname), '-u', settings.AWX_ISOLATED_USERNAME, '-T5', '-m', 'shell', - '-a', 'hostname', '-vvv' + '-a', 'awx-expect -h', '-vvv' ] if all([ getattr(settings, 'AWX_ISOLATED_KEY_GENERATION', False) is True, diff --git a/awx/main/migrations/0006_v320_release.py b/awx/main/migrations/0006_v320_release.py index c0a4330e04..8902a34438 100644 --- a/awx/main/migrations/0006_v320_release.py +++ b/awx/main/migrations/0006_v320_release.py @@ -484,7 +484,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name='instance', name='last_isolated_check', - field=models.DateTimeField(auto_now_add=True, null=True), + field=models.DateTimeField(editable=False, null=True), ), # Migrations that don't change db schema but simply to make Django ORM happy. # e.g. Choice updates, help_text updates, etc. diff --git a/awx/main/models/ha.py b/awx/main/models/ha.py index df53e1cd63..6386857f08 100644 --- a/awx/main/models/ha.py +++ b/awx/main/models/ha.py @@ -50,7 +50,6 @@ class Instance(BaseModel): last_isolated_check = models.DateTimeField( null=True, editable=False, - auto_now_add=True ) version = models.CharField(max_length=24, blank=True) capacity = models.PositiveIntegerField( diff --git a/awx/main/tasks.py b/awx/main/tasks.py index c0667d15b6..7edfcc7246 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -226,6 +226,7 @@ def handle_ha_toplogy_worker_ready(sender, **kwargs): # Expedite the first hearbeat run so a node comes online quickly. cluster_node_heartbeat.apply([]) apply_cluster_membership_policies.apply([]) + awx_isolated_heartbeat.apply([]) @celeryd_after_setup.connect @@ -380,7 +381,11 @@ def awx_isolated_heartbeat(self): accept_before = nowtime - timedelta(seconds=(poll_interval - 10)) isolated_instance_qs = Instance.objects.filter( rampart_groups__controller__instances__hostname=local_hostname, + ) + isolated_instance_qs = isolated_instance_qs.filter( last_isolated_check__lt=accept_before + ) | isolated_instance_qs.filter( + last_isolated_check=None ) # Fast pass of isolated instances, claiming the nodes to update with transaction.atomic(): @@ -883,6 +888,7 @@ class BaseTask(Task): stdout_handle = None try: + kwargs['isolated'] = instance.is_isolated() self.pre_run_hook(instance, **kwargs) if instance.cancel_flag: instance = self.update_model(instance.pk, status='canceled') diff --git a/awx/playbooks/clean_isolated.yml b/awx/playbooks/clean_isolated.yml index f904abe5a8..205dd7199e 100644 --- a/awx/playbooks/clean_isolated.yml +++ b/awx/playbooks/clean_isolated.yml @@ -18,7 +18,7 @@ file: path="{{item}}" state=absent register: result with_items: "{{cleanup_dirs}}" - until: result|succeeded + until: result is succeeded ignore_errors: yes retries: 3 delay: 5 @@ -26,4 +26,4 @@ - name: fail if build artifacts were not cleaned fail: msg: 'Unable to cleanup build artifacts' - when: not result|succeeded + when: not result is succeeded diff --git a/tools/docker-isolated-override.yml b/tools/docker-isolated-override.yml index 55ff09e97d..9f9d473def 100644 --- a/tools/docker-isolated-override.yml +++ b/tools/docker-isolated-override.yml @@ -14,6 +14,4 @@ services: - "../awx/main/expect:/awx_devel" - "../awx/lib:/awx_lib" - "/sys/fs/cgroup:/sys/fs/cgroup:ro" - tmpfs: - - "/tmp:exec" - - "/run" + privileged: true diff --git a/tools/docker-isolated/Dockerfile b/tools/docker-isolated/Dockerfile index 53a8b67481..69af7526cc 100644 --- a/tools/docker-isolated/Dockerfile +++ b/tools/docker-isolated/Dockerfile @@ -27,7 +27,4 @@ RUN ssh-keygen -A RUN mkdir -p /root/.ssh RUN touch /root/.ssh/authorized_keys -STOPSIGNAL SIGRTMIN+3 - - CMD ["/usr/sbin/init"]