Support for executing job and adhoc commands on isolated Tower nodes (#6524)

tools/docker-isolated/Dockerfile

@@ -0,0 +1,31 @@
+FROM centos/systemd
+RUN yum clean all
+
+ADD Makefile /tmp/Makefile
+RUN mkdir /tmp/requirements
+ADD requirements/requirements_ansible.txt requirements/requirements_ansible_git.txt requirements/requirements_ansible_uninstall.txt requirements/requirements_isolated.txt /tmp/requirements/
+RUN yum -y update && yum -y install curl epel-release
+RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash -
+RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git python-devel python-psycopg2 make python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel bubblewrap zanata-python-client gettext gcc-c++
+RUN pip install virtualenv
+WORKDIR /tmp
+RUN make requirements_ansible
+RUN make requirements_isolated
+RUN localedef -c -i en_US -f UTF-8 en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+WORKDIR /
+EXPOSE 22
+ADD tools/docker-isolated/playbook@.service /lib/systemd/system/playbook@.service
+
+RUN rm -f /etc/ssh/ssh_host_ecdsa_key /etc/ssh/ssh_host_rsa_key
+RUN ssh-keygen -q -N "" -t dsa -f /etc/ssh/ssh_host_ecdsa_key
+RUN ssh-keygen -q -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key
+RUN sed -i "s/#UsePrivilegeSeparation.*/UsePrivilegeSeparation no/g" /etc/ssh/sshd_config
+RUN sed -i "s/UsePAM.*/UsePAM yes/g" /etc/ssh/sshd_config
+RUN ssh-keygen -A
+RUN mkdir -p /root/.ssh
+RUN touch /root/.ssh/authorized_keys
+
+CMD ["/usr/sbin/init"]

tools/docker-isolated/Procfile

@@ -0,0 +1,8 @@
+nginx:          make nginx
+runworker:      make runworker
+daphne:         make daphne
+celeryd:        make celeryd EXTRA_GROUP_QUEUES=thepentagon
+receiver:       make receiver
+factcacher:     make factcacher
+flower:         make flower
+uwsgi:          make uwsgi

tools/docker-isolated/README.md

@@ -0,0 +1,71 @@
+## Instructions on using an isolated node
+
+The building of the isolated node is done in the `make docker-compose-build`
+target. Its image uses a different tag from the tools_tower container.
+
+Given that the images are built, you can run the combined docker compose target. This uses
+the base `docker-compose.yml` with modifications found in `docker-isolated-override.yml`.
+You will still need to give COMPOSE_TAG with whatever your intended
+base branch is. For example:
+
+```bash
+make docker-isolated COMPOSE_TAG=devel
+```
+
+This will automatically exchange the keys in order for the `tools_tower_1`
+container to access the `tools_isolated_1` container over ssh.
+After that, it will bring up all the containers like the normal docker-compose
+workflow.
+
+### Running a job on the Isolated Node
+
+Create a job template that runs normally. Add the id of the instance
+group named `thepentagon` to the JT's instance groups. To do this, POST
+the id (probably id=2) to `/api/v2/job_templates/N/instance_groups/`.
+After that, run the job template.
+
+The models are automatically created when running the Makefile target,
+and they are structured as follows:
+
+    +-------+     +-------------+
+    | tower |<----+ thepentagon |
+    +-------+     +-------------+
+        ^                ^
+        |                |
+        |                |
+    +---+---+      +-----+----+
+    | tower |      | isolated |
+    +-------+      +----------+
+
+The `controller` for the group "thepentagon" and all hosts therein is
+determined by a ForeignKey within the instance group.
+
+## Development Testing Notes
+
+### Test the SSH connection between containers
+
+While the environment is running, you can test the connection like so:
+
+```bash
+docker exec -i -t tools_tower_1 /bin/bash
+```
+
+Inside the context of that container:
+
+```bash
+ssh root@isolated
+```
+
+(note: awx user has been deprecated)
+
+This should give a shell to the `tools_isolated_1` container, as the
+`tools_tower_1` container sees it.
+
+### Start the playbook service
+
+The following command would run the playbook for job 57.
+
+```bash
+systemctl start playbook@57.service
+```
+

tools/docker-isolated/playbook@.service

@@ -0,0 +1,7 @@
+[Unit]
+Description=Run of Ansible Tower job %I
+
+[Service]
+ExecStart=/venv/tower_isolated/bin/python /tower_isolated/run.py %I
+Restart=no
+Environment=TOWER_LIB_DIRECTORY=/tower_lib