Changes for Tower virtualenv support

* Break requirements down into ansible and tower reqs
* Generate separate Ansible and Tower virtual environments
* Install appropriate requirements files into each one
* Modify development tools to use these venvs instead of our old
  site-packages
* Modify settings to indicate venv enablement and location of venvs
* Modify tasks to use the proper virtual environment for its purpose

.gitignore

@@ -97,3 +97,4 @@ reports
 # AWX python libs populated by requirements.txt
 awx/lib/.deps_built
 awx/lib/site-packages
+venv/*

Makefile

@@ -15,6 +15,8 @@ NPM_BIN ?= npm
 DEPS_SCRIPT ?= packaging/bundle/deps.py
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
 
+VENV_BASE ?= /tower_devel/venv
+
 CLIENT_TEST_DIR ?= build_test
 
 # Determine appropriate shasum command
@@ -232,24 +234,49 @@ rebase:
 push:
 	git push origin master
 
-# Install runtime, development and jenkins requirements
+virtualenv:
-requirements requirements_dev requirements_jenkins: %: real-%
+	if [ "$(VENV_BASE)" ]; then \
-
+		if [ ! -d "$(VENV_BASE)" ]; then \
-# Install third-party requirements needed for development environment.
+			mkdir $(VENV_BASE); \
-# NOTE:
+		fi; \
-#  * --target is only supported on newer versions of pip
+		if [ ! -d "$(VENV_BASE)/tower" ]; then \
-#  * https://github.com/pypa/pip/issues/3056 - the workaround is to override the `install-platlib`
+			virtualenv --system-site-packages $(VENV_BASE)/tower; \
-#  * --user (in conjunction with PYTHONUSERBASE="awx" may be a better option
+		fi; \
-#  * --target implies --ignore-installed
+		if [ ! -d "$(VENV_BASE)/ansible" ]; then \
-real-requirements:
+			virtualenv --system-site-packages $(VENV_BASE)/ansible; \
-	@if [ "$(PYTHON_VERSION)" = "2.6" ]; then \
+		fi; \
-	  pip install -r requirements/requirements_python26.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"; \
-	else \
-	  pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"; \
 	fi
 
-real-requirements_dev:
+# Install runtime, development and jenkins requirements
-	pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"
+requirements requirements_ansible requirements_dev requirements_jenkins: %: real-%
+
+real-requirements_ansible: virtualenv
+	if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/ansible/bin/activate; \
+	fi; \
+	pip install -r requirements/requirements_ansible.txt; \
+	if [ "$(VENV_BASE)" ]; then \
+		deactivate; \
+	fi
+
+# Install third-party requirements needed for Tower's environment.
+real-requirements: requirements_ansible
+	if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
+	pip install -r requirements/requirements.txt; \
+	if [ "$(VENV_BASE)" ]; then \
+		deactivate; \
+	fi
+
+real-requirements_dev: requirements_ansible
+	if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
+	pip install -r requirements/requirements_dev.txt; \
+	if [ "$(VENV_BASE)" ]; then \
+		deactivate; \
+	fi
 
 # Install third-party requirements needed for running unittests in jenkins
 real-requirements_jenkins:
@@ -272,11 +299,10 @@ version_file:
 
 # Do any one-time init tasks.
 init:
-	@if [ "$(VIRTUAL_ENV)" ]; then \
+	if [ "$(VENV_BASE)" ]; then \
-	    awx-manage register_instance --primary --hostname=127.0.0.1; \
+		. $(VENV_BASE)/tower/bin/activate; \
-	else \
+	fi; \
-	    sudo awx-manage register_instance --primary --hostname=127.0.0.1; \
+	tower-manage register_instance --primary --hostname=127.0.0.1
-	fi
 
 # Refresh development environment after pulling new code.
 refresh: clean requirements_dev version_file develop migrate
@@ -287,6 +313,9 @@ adduser:
 
 # Create database tables and apply any new migrations.
 migrate:
+	if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py migrate --noinput --fake-initial
 
 # Run after making changes to the models to create a new migration.
@@ -319,27 +348,48 @@ servercc: server_noattach
 # Alternate approach to tmux to run all development tasks specified in
 # Procfile.  https://youtu.be/OPMgaibszjk
 honcho:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	honcho start
 
 # Run the built-in development webserver (by default on http://localhost:8013).
 runserver:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py runserver
 
 # Run to start the background celery worker for development.
 celeryd:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py celeryd -l DEBUG -B --autoscale=20,2 -Ofair
 
 # Run to start the zeromq callback receiver
 receiver:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py run_callback_receiver
 
 taskmanager:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py run_task_system
 
 socketservice:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py run_socketio_service
 
 factcacher:
+	@if [ "$(VENV_BASE)" ]; then \
+		. $(VENV_BASE)/tower/bin/activate; \
+	fi; \
 	$(PYTHON) manage.py run_fact_cache_receiver
 
 reports:

awx/__init__.py

@@ -37,18 +37,6 @@ def find_commands(management_dir):
 def prepare_env():
     # Update the default settings environment variable based on current mode.
     os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE)
-    # Add local site-packages directory to path.
-    local_site_packages = os.path.join(os.path.dirname(__file__), 'lib',
-                                       'site-packages')
-    site.addsitedir(local_site_packages)
-    try:
-        index = sys.path.index(local_site_packages)
-        sys.path.pop(index)
-        # Work around https://bugs.python.org/issue7744
-        # by moving  local_site_packages to the front of sys.path
-        sys.path.insert(0, local_site_packages)
-    except ValueError:
-        pass
     # Hide DeprecationWarnings when running in production.  Need to first load
     # settings to apply our filter after Django's own warnings filter.
     from django.conf import settings

awx/main/tasks.py

@@ -445,11 +445,9 @@ class BaseTask(Task):
         # Set environment variables needed for inventory and job event
         # callbacks to work.
         # Update PYTHONPATH to use local site-packages.
-        python_paths = env.get('PYTHONPATH', '').split(os.pathsep)
+        if settings.ANSIBLE_USE_VENV:
-        local_site_packages = self.get_path_to('..', 'lib', 'site-packages')
+            env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH
-        if local_site_packages not in python_paths:
+            env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH']
-            python_paths.insert(0, local_site_packages)
-        env['PYTHONPATH'] = os.pathsep.join(python_paths)
         if self.should_use_proot:
             env['PROOT_TMP_DIR'] = tower_settings.AWX_PROOT_BASE_PATH
         return env
@@ -1276,7 +1274,9 @@ class RunInventoryUpdate(BaseTask):
         """
         env = super(RunInventoryUpdate, self).build_env(inventory_update,
                                                         **kwargs)
-
+        if settings.TOWER_USE_VENV:
+            env['VIRTUAL_ENV'] = settings.TOWER_VENV_PATH
+            env['PATH'] = os.path.join(settings.TOWER_VENV_PATH, "bin") + ":" + env['PATH']
         # Pass inventory source ID to inventory script.
         env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id)
         env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk)

awx/main/utils.py

@@ -487,6 +487,10 @@ def wrap_args_with_proot(args, cwd, **kwargs):
         show_paths = [cwd, kwargs['private_data_dir']]
     else:
         show_paths = [cwd]
+    if settings.ANSIBLE_USE_VENV:
+        show_paths.append(settings.ANSIBLE_VENV_PATH)
+    if settings.TOWER_USE_VENV:
+        show_paths.append(settings.TOWER_VENV_PATH)
     show_paths.extend(getattr(tower_settings, 'AWX_PROOT_SHOW_PATHS', None) or [])
     for path in sorted(set(show_paths)):
         if not os.path.exists(path):

awx/settings/development.py

@@ -79,6 +79,10 @@ STATSD_MAXUDPSIZE = 512
 include(optional('/etc/tower/settings.py'), scope=locals())
 include(optional('/etc/tower/conf.d/*.py'), scope=locals())
 
+ANSIBLE_USE_VENV = True
+ANSIBLE_VENV_PATH = "/tower_devel/venv/ansible"
+TOWER_USE_VENV = True
+TOWER_VENV_PATH = "/tower_devel/venv/tower"
 
 # If any local_*.py files are present in awx/settings/, use them to override
 # default settings for development.  If not present, we can still run using

awx/settings/production.py

@@ -41,6 +41,14 @@ JOBOUTPUT_ROOT = '/var/lib/awx/job_status/'
 # The heartbeat file for the tower scheduler
 SCHEDULE_METADATA_LOCATION = '/var/lib/awx/.tower_cycle'
 
+# Ansible base virtualenv paths and enablement
+ANSIBLE_USE_VENV = True
+ANSIBLE_VENV_PATH = "/var/lib/awx/venv/ansible"
+
+# Tower base virtualenv paths and enablement
+TOWER_USE_VENV = True
+TOWER_VENV_PATH = "/var/lib/awx/venv/tower"
+
 LOGGING['handlers']['tower_warnings'] = {
     'level': 'WARNING',
     'class':'logging.handlers.RotatingFileHandler',

requirements/requirements.txt

@@ -12,9 +12,9 @@ cffi==1.5.0
 cliff==1.15.0
 cmd2==0.6.8
 cryptography==0.9.3
-d2to1==0.2.11
+d2to1==0.2.11   # TODO: Still needed?
 defusedxml==0.4.1
-Django==1.8.8
+Django==1.8.10
 debtcollector==1.2.0
 decorator==4.0.6
 django-auth-ldap==1.2.6
@@ -31,15 +31,12 @@ django-statsd-mozilla==0.3.16
 django-taggit==0.17.6
 git+https://github.com/matburt/dm.xmlsec.binding.git@master#egg=dm.xmlsec.binding
 dogpile.core==0.4.1
-#functools32==3.2.3-2
 gevent==1.1rc3
 gevent-websocket==0.9.3
 git+https://github.com/chrismeyersfsu/django-jsonfield.git@tower_0.9.12#egg=django-jsonfield
 git+https://github.com/chrismeyersfsu/django-qsstats-magic.git@tower_0.7.2#egg=django-qsstats-magic
 git+https://github.com/umutbozkurt/django-rest-framework-mongoengine.git@5dfa1df79f81765d36c0de31dc1c2f390e42d428#egg=django-rest-framework-mongoengine
 git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=gevent-socketio
-git+https://github.com/chrismeyersfsu/python-ipy.git@fix-127_localhost#egg=IPy
-git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize
 greenlet==0.4.9
 dogpile.cache==0.5.7
 enum34==1.1.2
@@ -82,7 +79,7 @@ pexpect==3.1
 prettytable==0.7.2
 psphere==0.5.2
 psutil==3.1.1
-psycopg2
+psycopg2==2.6.1
 pyasn1==0.1.9
 pycrypto==2.6.1
 pycparser==2.14

requirements/requirements_ansible.txt

@@ -0,0 +1,79 @@
+anyjson==0.3.3
+apache-libcloud==0.15.1
+appdirs==1.4.0
+azure==0.9.0
+Babel==2.2.0
+boto==2.34.0
+cliff==1.15.0
+cmd2==0.6.8
+cryptography==0.9.3
+debtcollector==1.2.0
+decorator==4.0.6
+dogpile.core==0.4.1
+dogpile.cache==0.5.7
+futures==3.0.4
+httplib2==0.9.2
+idna==2.0
+importlib==1.0.3
+ip-associations-python-novaclient-ext==0.1
+ipaddress==1.0.16
+iso8601==0.1.11
+isodate==0.5.1
+jsonpatch==1.12
+jsonpointer==1.10
+jsonschema==2.5.1
+keyring==4.1
+lxml==3.4.4
+mock==1.0.1
+monotonic==0.6
+msgpack-python==0.4.7
+munch==2.0.4
+netaddr==0.7.18
+netifaces==0.10.4
+os-client-config==1.14.0
+os-diskconfig-python-novaclient-ext==0.1.3
+os-networksv2-python-novaclient-ext==0.25
+os-virtual-interfacesv2-python-novaclient-ext==0.19
+pbr==0.11.1
+oslo.config==3.3.0
+oslo.i18n==3.2.0
+oslo.serialization==2.2.0
+oslo.utils==3.4.0
+prettytable==0.7.2
+psphere==0.5.2
+pyasn1==0.1.9
+pycrypto==2.6.1
+pycparser==2.14
+pyOpenSSL==0.15.1
+pyparsing==2.0.7
+pyrax==1.9.7
+python-cinderclient==1.5.0
+python-dateutil==2.4.0
+python-glanceclient==1.1.0
+python-heatclient==0.8.1
+python-ironicclient==1.0.0
+python-keystoneclient==2.1.1
+python-neutronclient==4.0.0
+python-novaclient==3.2.0
+python-openstackclient==2.0.0
+python-swiftclient==2.7.0
+python-troveclient==1.4.0
+pytz==2015.7
+pywinrm==0.1.1
+PyYAML==3.11
+pyzmq==14.5.0
+rackspace-auth-openstack==1.3
+rackspace-novaclient==1.5
+rax-default-network-flags-python-novaclient-ext==0.3.2
+rax-scheduled-images-python-novaclient-ext==0.3.1
+requests==2.5.1
+requestsexceptions==1.1.1
+shade==1.4.0
+simplejson==3.8.1
+six==1.9.0
+stevedore==1.10.0
+suds==0.4
+unicodecsv==0.14.1
+warlock==1.2.0
+wrapt==1.10.6
+xmltodict==0.9.2

requirements/requirements_jenkins.txt

@@ -1,7 +1,4 @@
--r requirements.txt
 ansible==1.9.4
-# Based on django-jenkins==0.16.3, with a fix for properly importing coverage
-git+https://github.com/jlaska/django-jenkins.git@release_0.16.4#egg=django-jenkins
 coverage
 pyflakes
 pep8

requirements/requirements_python26.txt

@@ -1,121 +0,0 @@
-amqp==1.4.5
-git+https://github.com/chrismeyersfsu/ansiconv.git@tower_1.0.0#egg=ansiconv
-anyjson==0.3.3
-apache-libcloud==0.15.1
-appdirs==1.4.0
-argparse==1.2.1
-azure==0.9.0
-Babel==1.3
-billiard==3.3.0.16
-boto==2.34.0
-celery==3.1.10
-cffi==1.1.2
-cliff==1.13.0
-cmd2==0.6.8
-cryptography==0.9.3
-d2to1==0.2.11
-defusedxml==0.4.1
-Django==1.6.7
-django-auth-ldap==1.2.6
-django-celery==3.1.10
-django-crum==0.6.1
-django-extensions==1.3.3
-django-polymorphic==0.5.3
-django-radius==1.0.0
-djangorestframework==2.3.13
-django-split-settings==0.1.1
-django-taggit==0.11.2
-git+https://github.com/matburt/dm.xmlsec.binding.git@master#egg=dm.xmlsec.binding
-dogpile.cache==0.5.6
-dogpile.core==0.4.1
-enum34==1.0.4
-#functools32==3.2.3-2
-gevent==1.1rc3
-gevent-websocket==0.9.3
-git+https://github.com/chrismeyersfsu/django-jsonfield.git@tower_0.9.12#egg=django-jsonfield
-git+https://github.com/chrismeyersfsu/django-qsstats-magic.git@tower_0.7.2#egg=django-qsstats-magic
-git+https://github.com/chrismeyersfsu/django-rest-framework-mongoengine.git@0c79515257a33a0ce61500b65fa497398628a03d#egg=django-rest-framework-mongoengine
-git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=gevent-socketio
-git+https://github.com/chrismeyersfsu/python-ipy.git@fix-127_localhost#egg=IPy
-git+https://github.com/chrismeyersfsu/python-keystoneclient.git@1.3.0#egg=python-keystoneclient
-git+https://github.com/chrismeyersfsu/shade.git@tower_0.5.0#egg=shade
-git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize
-greenlet==0.4.9
-httplib2==0.9
-idna==2.0
-importlib==1.0.3
-ipaddress==1.0.14
-iso8601==0.1.10
-isodate==0.5.1
-jsonpatch==1.11
-jsonpointer==1.9
-jsonschema==2.5.1
-keyring==4.1
-kombu==3.0.21
-lxml==3.4.4
-M2Crypto==0.22.3
-Markdown==2.4.1
-mock==1.0.1
-mongoengine==0.9.0
-msgpack-python==0.4.6
-netaddr==0.7.14
-netifaces==0.10.4
-oauthlib==1.0.3
-ordereddict==1.1
-os-client-config==1.6.1
-os-diskconfig-python-novaclient-ext==0.1.2
-oslo.config==1.9.3
-oslo.i18n==1.5.0
-oslo.serialization==1.4.0
-oslo.utils==1.4.0
-os-networksv2-python-novaclient-ext==0.25
-os-virtual-interfacesv2-python-novaclient-ext==0.19
-pbr==0.11.1
-pexpect==3.1
-pip==1.5.4
-prettytable==0.7.2
-psphere==0.5.2
-psutil==3.1.1
-psycopg2
-pyasn1==0.1.8
-pycparser==2.14
-pycrypto==2.6.1
-PyJWT==1.4.0
-pymongo==2.8
-pyOpenSSL==0.15.1
-pyparsing==2.0.3
-pyrad==2.0
-pyrax==1.9.3
-python-cinderclient==1.1.1
-python-dateutil==2.4.0
-python-glanceclient==0.17.0
-python-ironicclient==0.5.0
-python-ldap==2.4.20
-python-neutronclient==2.3.11
-python-novaclient==2.20.0
-python-openid==2.2.5
-python-radius==1.0
-git+https://github.com/matburt/python-social-auth.git@master#egg=python-social-auth
-python-saml==2.1.4
-python-swiftclient==2.2.0
-python-troveclient==1.0.9
-pytz==2014.10
-pywinrm==0.1.1
-PyYAML==3.11
-pyzmq==14.5.0
-rackspace-auth-openstack==1.3
-rackspace-novaclient==1.4
-rax-default-network-flags-python-novaclient-ext==0.2.3
-rax-scheduled-images-python-novaclient-ext==0.2.1
-redis==2.10.3
-requests==2.5.1
-requests-oauthlib==0.5.0
-simplejson==3.6.0
-six==1.9.0
-South==1.0.2
-stevedore==1.3.0
-suds==0.4
-warlock==1.1.0
-wheel==0.24.0
-wsgiref==0.1.2
-xmltodict==0.9.2

tools/docker-compose/Dockerfile

@@ -11,9 +11,7 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt
 RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list
 RUN apt-get update
 RUN apt-get install -y openssh-server ansible mg vim tmux git mercurial subversion python-dev python-psycopg2 make postgresql-client libpq-dev nodejs python-psutil libxml2-dev libxslt-dev lib32z1-dev libsasl2-dev libldap2-dev libffi-dev libzmq-dev proot python-pip libxmlsec1-dev swig redis-server && rm -rf /var/lib/apt/lists/*
-RUN pip install flake8
+RUN pip install flake8 pytest pytest-pythonpath pytest-django pytest-cov pytest-mock dateutils django-debug-toolbar==1.4 pyflakes==1.0.0 virtualenv
-RUN pip install pytest pytest-pythonpath pytest-django pytest-cov
-RUN pip install dateutils # for private/license_writer.py
 RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa
 RUN mkdir -p /etc/tower
 RUN mkdir -p /data/db
@@ -23,6 +21,9 @@ RUN pip2 install honcho
 RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.0.0/dumb-init_1.0.0_amd64.deb
 RUN dpkg -i dumb-init_*.deb
 ADD start_development.sh /start_development.sh
+ADD ansible-tower.egg-link /tmp/ansible-tower.egg-link
+ADD tower-manage /usr/local/bin/tower-manage
+ADD ansible_tower.egg-info /tmp/ansible_tower.egg-info
 
 EXPOSE 8013 8080 22
 ENTRYPOINT ["/usr/bin/dumb-init"]

tools/docker-compose/start_development.sh

@@ -20,12 +20,8 @@ else
     echo "Failed to find tower source tree, map your development tree volume"
 fi
 
-if [ -f "/.develop_run" ]; then
+rm -rf /tower_devel/ansible_tower.egg-info
-    echo "Skipping 'make develop' step since it has already run - remove /.develop_run to force it"
+cp -R /tmp/ansible_tower.egg-info /tower_devel/
-else
-    make develop
-    touch /.develop_run
-fi
 
 # Check if we need to build dependencies
 if [ -f "awx/lib/.deps_built" ]; then
@@ -35,6 +31,9 @@ else
     touch awx/lib/.deps_built
 fi
 
+rm -rf /tower_devel/venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link
+cp /tmp/ansible-tower.egg-link /tower_devel/venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link
+
 # Tower bootstrapping
 make version_file
 make migrate