diff --git a/.gitignore b/.gitignore index c385ad667d..095072596e 100644 --- a/.gitignore +++ b/.gitignore @@ -97,3 +97,4 @@ reports # AWX python libs populated by requirements.txt awx/lib/.deps_built awx/lib/site-packages +venv/* diff --git a/Makefile b/Makefile index a6461e8e66..1f577e3164 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,8 @@ NPM_BIN ?= npm DEPS_SCRIPT ?= packaging/bundle/deps.py GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) +VENV_BASE ?= /tower_devel/venv + CLIENT_TEST_DIR ?= build_test # Determine appropriate shasum command @@ -232,24 +234,49 @@ rebase: push: git push origin master -# Install runtime, development and jenkins requirements -requirements requirements_dev requirements_jenkins: %: real-% - -# Install third-party requirements needed for development environment. -# NOTE: -# * --target is only supported on newer versions of pip -# * https://github.com/pypa/pip/issues/3056 - the workaround is to override the `install-platlib` -# * --user (in conjunction with PYTHONUSERBASE="awx" may be a better option -# * --target implies --ignore-installed -real-requirements: - @if [ "$(PYTHON_VERSION)" = "2.6" ]; then \ - pip install -r requirements/requirements_python26.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"; \ - else \ - pip install -r requirements/requirements.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python"; \ +virtualenv: + if [ "$(VENV_BASE)" ]; then \ + if [ ! -d "$(VENV_BASE)" ]; then \ + mkdir $(VENV_BASE); \ + fi; \ + if [ ! -d "$(VENV_BASE)/tower" ]; then \ + virtualenv --system-site-packages $(VENV_BASE)/tower; \ + fi; \ + if [ ! -d "$(VENV_BASE)/ansible" ]; then \ + virtualenv --system-site-packages $(VENV_BASE)/ansible; \ + fi; \ fi -real-requirements_dev: - pip install -r requirements/requirements_dev.txt --target awx/lib/site-packages/ --install-option="--install-platlib=\$$base/lib/python" +# Install runtime, development and jenkins requirements +requirements requirements_ansible requirements_dev requirements_jenkins: %: real-% + +real-requirements_ansible: virtualenv + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/ansible/bin/activate; \ + fi; \ + pip install -r requirements/requirements_ansible.txt; \ + if [ "$(VENV_BASE)" ]; then \ + deactivate; \ + fi + +# Install third-party requirements needed for Tower's environment. +real-requirements: requirements_ansible + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + pip install -r requirements/requirements.txt; \ + if [ "$(VENV_BASE)" ]; then \ + deactivate; \ + fi + +real-requirements_dev: requirements_ansible + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + pip install -r requirements/requirements_dev.txt; \ + if [ "$(VENV_BASE)" ]; then \ + deactivate; \ + fi # Install third-party requirements needed for running unittests in jenkins real-requirements_jenkins: @@ -272,7 +299,10 @@ version_file: # Do any one-time init tasks. init: - @if [ "$(VIRTUAL_ENV)" ]; then \ + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ + if [ "$(VIRTUAL_ENV)" ]; then \ tower-manage register_instance --primary --hostname=127.0.0.1; \ else \ sudo tower-manage register_instance --primary --hostname=127.0.0.1; \ @@ -287,6 +317,9 @@ adduser: # Create database tables and apply any new migrations. migrate: + if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py migrate --noinput --fake-initial # Run after making changes to the models to create a new migration. @@ -319,27 +352,48 @@ servercc: server_noattach # Alternate approach to tmux to run all development tasks specified in # Procfile. https://youtu.be/OPMgaibszjk honcho: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ honcho start # Run the built-in development webserver (by default on http://localhost:8013). runserver: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py runserver # Run to start the background celery worker for development. celeryd: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py celeryd -l DEBUG -B --autoscale=20,2 -Ofair # Run to start the zeromq callback receiver receiver: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py run_callback_receiver taskmanager: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py run_task_system socketservice: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py run_socketio_service factcacher: + @if [ "$(VENV_BASE)" ]; then \ + . $(VENV_BASE)/tower/bin/activate; \ + fi; \ $(PYTHON) manage.py run_fact_cache_receiver reports: diff --git a/awx/__init__.py b/awx/__init__.py index d597dff532..f498ae5553 100644 --- a/awx/__init__.py +++ b/awx/__init__.py @@ -37,18 +37,6 @@ def find_commands(management_dir): def prepare_env(): # Update the default settings environment variable based on current mode. os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'awx.settings.%s' % MODE) - # Add local site-packages directory to path. - local_site_packages = os.path.join(os.path.dirname(__file__), 'lib', - 'site-packages') - site.addsitedir(local_site_packages) - try: - index = sys.path.index(local_site_packages) - sys.path.pop(index) - # Work around https://bugs.python.org/issue7744 - # by moving local_site_packages to the front of sys.path - sys.path.insert(0, local_site_packages) - except ValueError: - pass # Hide DeprecationWarnings when running in production. Need to first load # settings to apply our filter after Django's own warnings filter. from django.conf import settings diff --git a/awx/main/tasks.py b/awx/main/tasks.py index 509c5d1e7e..2acd48a42e 100644 --- a/awx/main/tasks.py +++ b/awx/main/tasks.py @@ -445,11 +445,9 @@ class BaseTask(Task): # Set environment variables needed for inventory and job event # callbacks to work. # Update PYTHONPATH to use local site-packages. - python_paths = env.get('PYTHONPATH', '').split(os.pathsep) - local_site_packages = self.get_path_to('..', 'lib', 'site-packages') - if local_site_packages not in python_paths: - python_paths.insert(0, local_site_packages) - env['PYTHONPATH'] = os.pathsep.join(python_paths) + if settings.ANSIBLE_USE_VENV: + env['VIRTUAL_ENV'] = settings.ANSIBLE_VENV_PATH + env['PATH'] = os.path.join(settings.ANSIBLE_VENV_PATH, "bin") + ":" + env['PATH'] if self.should_use_proot: env['PROOT_TMP_DIR'] = tower_settings.AWX_PROOT_BASE_PATH return env @@ -1276,7 +1274,9 @@ class RunInventoryUpdate(BaseTask): """ env = super(RunInventoryUpdate, self).build_env(inventory_update, **kwargs) - + if settings.TOWER_USE_VENV: + env['VIRTUAL_ENV'] = settings.TOWER_VENV_PATH + env['PATH'] = os.path.join(settings.TOWER_VENV_PATH, "bin") + ":" + env['PATH'] # Pass inventory source ID to inventory script. env['INVENTORY_SOURCE_ID'] = str(inventory_update.inventory_source_id) env['INVENTORY_UPDATE_ID'] = str(inventory_update.pk) diff --git a/awx/main/utils.py b/awx/main/utils.py index a561648f95..b91df7ec34 100644 --- a/awx/main/utils.py +++ b/awx/main/utils.py @@ -487,6 +487,10 @@ def wrap_args_with_proot(args, cwd, **kwargs): show_paths = [cwd, kwargs['private_data_dir']] else: show_paths = [cwd] + if settings.ANSIBLE_USE_VENV: + show_paths.append(settings.ANSIBLE_VENV_PATH) + if settings.TOWER_USE_VENV: + show_paths.append(settings.TOWER_VENV_PATH) show_paths.extend(getattr(tower_settings, 'AWX_PROOT_SHOW_PATHS', None) or []) for path in sorted(set(show_paths)): if not os.path.exists(path): diff --git a/awx/settings/development.py b/awx/settings/development.py index 46df026e06..cc4febdbae 100644 --- a/awx/settings/development.py +++ b/awx/settings/development.py @@ -79,6 +79,10 @@ STATSD_MAXUDPSIZE = 512 include(optional('/etc/tower/settings.py'), scope=locals()) include(optional('/etc/tower/conf.d/*.py'), scope=locals()) +ANSIBLE_USE_VENV = True +ANSIBLE_VENV_PATH = "/tower_devel/venv/ansible" +TOWER_USE_VENV = True +TOWER_VENV_PATH = "/tower_devel/venv/tower" # If any local_*.py files are present in awx/settings/, use them to override # default settings for development. If not present, we can still run using diff --git a/awx/settings/production.py b/awx/settings/production.py index 766029f87a..5460061bc8 100644 --- a/awx/settings/production.py +++ b/awx/settings/production.py @@ -41,6 +41,14 @@ JOBOUTPUT_ROOT = '/var/lib/awx/job_status/' # The heartbeat file for the tower scheduler SCHEDULE_METADATA_LOCATION = '/var/lib/awx/.tower_cycle' +# Ansible base virtualenv paths and enablement +ANSIBLE_USE_VENV = True +ANSIBLE_VENV_PATH = "/var/lib/awx/venv/ansible" + +# Tower base virtualenv paths and enablement +TOWER_USE_VENV = True +TOWER_VENV_PATH = "/var/lib/awx/venv/tower" + LOGGING['handlers']['tower_warnings'] = { 'level': 'WARNING', 'class':'logging.handlers.RotatingFileHandler', diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 254d3c0237..d74cf1819e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -12,9 +12,9 @@ cffi==1.5.0 cliff==1.15.0 cmd2==0.6.8 cryptography==0.9.3 -d2to1==0.2.11 +d2to1==0.2.11 # TODO: Still needed? defusedxml==0.4.1 -Django==1.8.8 +Django==1.8.10 debtcollector==1.2.0 decorator==4.0.6 django-auth-ldap==1.2.6 @@ -31,15 +31,12 @@ django-statsd-mozilla==0.3.16 django-taggit==0.17.6 git+https://github.com/matburt/dm.xmlsec.binding.git@master#egg=dm.xmlsec.binding dogpile.core==0.4.1 -#functools32==3.2.3-2 gevent==1.1rc3 gevent-websocket==0.9.3 git+https://github.com/chrismeyersfsu/django-jsonfield.git@tower_0.9.12#egg=django-jsonfield git+https://github.com/chrismeyersfsu/django-qsstats-magic.git@tower_0.7.2#egg=django-qsstats-magic git+https://github.com/umutbozkurt/django-rest-framework-mongoengine.git@5dfa1df79f81765d36c0de31dc1c2f390e42d428#egg=django-rest-framework-mongoengine git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=gevent-socketio -git+https://github.com/chrismeyersfsu/python-ipy.git@fix-127_localhost#egg=IPy -git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize greenlet==0.4.9 dogpile.cache==0.5.7 enum34==1.1.2 @@ -82,7 +79,7 @@ pexpect==3.1 prettytable==0.7.2 psphere==0.5.2 psutil==3.1.1 -psycopg2 +psycopg2==2.6.1 pyasn1==0.1.9 pycrypto==2.6.1 pycparser==2.14 diff --git a/requirements/requirements_ansible.txt b/requirements/requirements_ansible.txt new file mode 100644 index 0000000000..adf76989e2 --- /dev/null +++ b/requirements/requirements_ansible.txt @@ -0,0 +1,79 @@ +anyjson==0.3.3 +apache-libcloud==0.15.1 +appdirs==1.4.0 +azure==0.9.0 +Babel==2.2.0 +boto==2.34.0 +cliff==1.15.0 +cmd2==0.6.8 +cryptography==0.9.3 +debtcollector==1.2.0 +decorator==4.0.6 +dogpile.core==0.4.1 +dogpile.cache==0.5.7 +futures==3.0.4 +httplib2==0.9.2 +idna==2.0 +importlib==1.0.3 +ip-associations-python-novaclient-ext==0.1 +ipaddress==1.0.16 +iso8601==0.1.11 +isodate==0.5.1 +jsonpatch==1.12 +jsonpointer==1.10 +jsonschema==2.5.1 +keyring==4.1 +lxml==3.4.4 +mock==1.0.1 +monotonic==0.6 +msgpack-python==0.4.7 +munch==2.0.4 +netaddr==0.7.18 +netifaces==0.10.4 +os-client-config==1.14.0 +os-diskconfig-python-novaclient-ext==0.1.3 +os-networksv2-python-novaclient-ext==0.25 +os-virtual-interfacesv2-python-novaclient-ext==0.19 +pbr==0.11.1 +oslo.config==3.3.0 +oslo.i18n==3.2.0 +oslo.serialization==2.2.0 +oslo.utils==3.4.0 +prettytable==0.7.2 +psphere==0.5.2 +pyasn1==0.1.9 +pycrypto==2.6.1 +pycparser==2.14 +pyOpenSSL==0.15.1 +pyparsing==2.0.7 +pyrax==1.9.7 +python-cinderclient==1.5.0 +python-dateutil==2.4.0 +python-glanceclient==1.1.0 +python-heatclient==0.8.1 +python-ironicclient==1.0.0 +python-keystoneclient==2.1.1 +python-neutronclient==4.0.0 +python-novaclient==3.2.0 +python-openstackclient==2.0.0 +python-swiftclient==2.7.0 +python-troveclient==1.4.0 +pytz==2015.7 +pywinrm==0.1.1 +PyYAML==3.11 +pyzmq==14.5.0 +rackspace-auth-openstack==1.3 +rackspace-novaclient==1.5 +rax-default-network-flags-python-novaclient-ext==0.3.2 +rax-scheduled-images-python-novaclient-ext==0.3.1 +requests==2.5.1 +requestsexceptions==1.1.1 +shade==1.4.0 +simplejson==3.8.1 +six==1.9.0 +stevedore==1.10.0 +suds==0.4 +unicodecsv==0.14.1 +warlock==1.2.0 +wrapt==1.10.6 +xmltodict==0.9.2 diff --git a/requirements/requirements_jenkins.txt b/requirements/requirements_jenkins.txt index 498a5040ad..ff3fda270f 100644 --- a/requirements/requirements_jenkins.txt +++ b/requirements/requirements_jenkins.txt @@ -1,7 +1,4 @@ --r requirements.txt ansible==1.9.4 -# Based on django-jenkins==0.16.3, with a fix for properly importing coverage -git+https://github.com/jlaska/django-jenkins.git@release_0.16.4#egg=django-jenkins coverage pyflakes==1.0.0 # Pinned until PR merges https://gitlab.com/pycqa/flake8/merge_requests/56 pep8 diff --git a/requirements/requirements_python26.txt b/requirements/requirements_python26.txt deleted file mode 100644 index 8158d952df..0000000000 --- a/requirements/requirements_python26.txt +++ /dev/null @@ -1,121 +0,0 @@ -amqp==1.4.5 -git+https://github.com/chrismeyersfsu/ansiconv.git@tower_1.0.0#egg=ansiconv -anyjson==0.3.3 -apache-libcloud==0.15.1 -appdirs==1.4.0 -argparse==1.2.1 -azure==0.9.0 -Babel==1.3 -billiard==3.3.0.16 -boto==2.34.0 -celery==3.1.10 -cffi==1.1.2 -cliff==1.13.0 -cmd2==0.6.8 -cryptography==0.9.3 -d2to1==0.2.11 -defusedxml==0.4.1 -Django==1.6.7 -django-auth-ldap==1.2.6 -django-celery==3.1.10 -django-crum==0.6.1 -django-extensions==1.3.3 -django-polymorphic==0.5.3 -django-radius==1.0.0 -djangorestframework==2.3.13 -django-split-settings==0.1.1 -django-taggit==0.11.2 -git+https://github.com/matburt/dm.xmlsec.binding.git@master#egg=dm.xmlsec.binding -dogpile.cache==0.5.6 -dogpile.core==0.4.1 -enum34==1.0.4 -#functools32==3.2.3-2 -gevent==1.1rc3 -gevent-websocket==0.9.3 -git+https://github.com/chrismeyersfsu/django-jsonfield.git@tower_0.9.12#egg=django-jsonfield -git+https://github.com/chrismeyersfsu/django-qsstats-magic.git@tower_0.7.2#egg=django-qsstats-magic -git+https://github.com/chrismeyersfsu/django-rest-framework-mongoengine.git@0c79515257a33a0ce61500b65fa497398628a03d#egg=django-rest-framework-mongoengine -git+https://github.com/chrismeyersfsu/gevent-socketio.git@tower_0.3.6#egg=gevent-socketio -git+https://github.com/chrismeyersfsu/python-ipy.git@fix-127_localhost#egg=IPy -git+https://github.com/chrismeyersfsu/python-keystoneclient.git@1.3.0#egg=python-keystoneclient -git+https://github.com/chrismeyersfsu/shade.git@tower_0.5.0#egg=shade -git+https://github.com/chrismeyersfsu/sitecustomize.git#egg=sitecustomize -greenlet==0.4.9 -httplib2==0.9 -idna==2.0 -importlib==1.0.3 -ipaddress==1.0.14 -iso8601==0.1.10 -isodate==0.5.1 -jsonpatch==1.11 -jsonpointer==1.9 -jsonschema==2.5.1 -keyring==4.1 -kombu==3.0.21 -lxml==3.4.4 -M2Crypto==0.22.3 -Markdown==2.4.1 -mock==1.0.1 -mongoengine==0.9.0 -msgpack-python==0.4.6 -netaddr==0.7.14 -netifaces==0.10.4 -oauthlib==1.0.3 -ordereddict==1.1 -os-client-config==1.6.1 -os-diskconfig-python-novaclient-ext==0.1.2 -oslo.config==1.9.3 -oslo.i18n==1.5.0 -oslo.serialization==1.4.0 -oslo.utils==1.4.0 -os-networksv2-python-novaclient-ext==0.25 -os-virtual-interfacesv2-python-novaclient-ext==0.19 -pbr==0.11.1 -pexpect==3.1 -pip==1.5.4 -prettytable==0.7.2 -psphere==0.5.2 -psutil==3.1.1 -psycopg2 -pyasn1==0.1.8 -pycparser==2.14 -pycrypto==2.6.1 -PyJWT==1.4.0 -pymongo==2.8 -pyOpenSSL==0.15.1 -pyparsing==2.0.3 -pyrad==2.0 -pyrax==1.9.3 -python-cinderclient==1.1.1 -python-dateutil==2.4.0 -python-glanceclient==0.17.0 -python-ironicclient==0.5.0 -python-ldap==2.4.20 -python-neutronclient==2.3.11 -python-novaclient==2.20.0 -python-openid==2.2.5 -python-radius==1.0 -git+https://github.com/matburt/python-social-auth.git@master#egg=python-social-auth -python-saml==2.1.4 -python-swiftclient==2.2.0 -python-troveclient==1.0.9 -pytz==2014.10 -pywinrm==0.1.1 -PyYAML==3.11 -pyzmq==14.5.0 -rackspace-auth-openstack==1.3 -rackspace-novaclient==1.4 -rax-default-network-flags-python-novaclient-ext==0.2.3 -rax-scheduled-images-python-novaclient-ext==0.2.1 -redis==2.10.3 -requests==2.5.1 -requests-oauthlib==0.5.0 -simplejson==3.6.0 -six==1.9.0 -South==1.0.2 -stevedore==1.3.0 -suds==0.4 -warlock==1.1.0 -wheel==0.24.0 -wsgiref==0.1.2 -xmltodict==0.9.2 diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 72568d3dfc..91c156e1ef 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -11,9 +11,7 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list RUN apt-get update RUN apt-get install -y openssh-server ansible mg vim tmux git mercurial subversion python-dev python-psycopg2 make postgresql-client libpq-dev nodejs python-psutil libxml2-dev libxslt-dev lib32z1-dev libsasl2-dev libldap2-dev libffi-dev libzmq-dev proot python-pip libxmlsec1-dev swig redis-server && rm -rf /var/lib/apt/lists/* -RUN pip install flake8 -RUN pip install pytest pytest-pythonpath pytest-django pytest-cov -RUN pip install dateutils # for private/license_writer.py +RUN pip install flake8 pytest pytest-pythonpath pytest-django pytest-cov pytest-mock dateutils django-debug-toolbar==1.4 pyflakes==1.0.0 virtualenv RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa RUN mkdir -p /etc/tower RUN mkdir -p /data/db diff --git a/tools/docker-compose/start_development.sh b/tools/docker-compose/start_development.sh index bfbf7cc17d..a8ae080a2b 100755 --- a/tools/docker-compose/start_development.sh +++ b/tools/docker-compose/start_development.sh @@ -21,7 +21,7 @@ else fi rm -rf /tower_devel/ansible_tower.egg-info -mv /tmp/ansible_tower.egg-info /tower_devel/ +cp -R /tmp/ansible_tower.egg-info /tower_devel/ # Check if we need to build dependencies if [ -f "awx/lib/.deps_built" ]; then @@ -31,6 +31,9 @@ else touch awx/lib/.deps_built fi +rm -rf /tower_devel/venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link +cp /tmp/ansible-tower.egg-link /tower_devel/venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link + # Tower bootstrapping make version_file make migrate