diff --git a/Makefile b/Makefile index 8076ba2faf..3154edbe09 100644 --- a/Makefile +++ b/Makefile @@ -10,7 +10,7 @@ NPM_BIN ?= npm DEPS_SCRIPT ?= packaging/bundle/deps.py GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD) -VENV_BASE ?= /tower_devel/venv +VENV_BASE ?= /venv SCL_PREFIX ?= CELERY_SCHEDULE_FILE ?= /celerybeat-schedule @@ -246,7 +246,7 @@ virtualenv_ansible: if [ ! -d "$(VENV_BASE)/ansible" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/ansible && \ $(VENV_BASE)/ansible/bin/pip install -I setuptools==23.0.0 && \ - $(VENV_BASE)/ansible/bin/pip install -I pip==8.1.1; \ + $(VENV_BASE)/ansible/bin/pip install -I pip==8.1.2; \ fi; \ fi @@ -258,7 +258,7 @@ virtualenv_tower: if [ ! -d "$(VENV_BASE)/tower" ]; then \ virtualenv --system-site-packages --setuptools $(VENV_BASE)/tower && \ $(VENV_BASE)/tower/bin/pip install -I setuptools==23.0.0 && \ - $(VENV_BASE)/tower/bin/pip install -I pip==8.1.1; \ + $(VENV_BASE)/tower/bin/pip install -I pip==8.1.2; \ fi; \ fi diff --git a/awx/main/queue.py b/awx/main/queue.py index 9b0b1f60b3..b1cc7694fd 100644 --- a/awx/main/queue.py +++ b/awx/main/queue.py @@ -3,43 +3,11 @@ import json -from redis import StrictRedis - from django.conf import settings __all__ = ['FifoQueue'] - -# Determine, based on settings.BROKER_URL (for celery), what the correct Redis -# connection settings are. -redis_kwargs = {} -broker_url = settings.BROKER_URL -if not broker_url.lower().startswith('redis://'): - raise RuntimeError('Error importing awx.main.queue: Cannot use queue with ' - 'a non-Redis broker configured for celery.\n' - 'Broker is set to: %s' % broker_url) -broker_url = broker_url[8:] - -# There may or may not be a password; address both situations by checking -# for an "@" in the broker URL. -if '@' in broker_url: - broker_auth, broker_host = broker_url.split('@') - redis_kwargs['password'] = broker_auth.split(':')[1] -else: - broker_host = broker_url - -# Ignore anything after a / in the broker host. -broker_host = broker_host.split('/')[0] - -# If a custom port is present, parse it out. -if ':' in broker_host: - broker_host, broker_port = broker_host.split(':') - redis_kwargs['port'] = int(broker_port) - -# Now create a StrictRedis object that knows how to connect appropriately. -redis = StrictRedis(broker_host, **redis_kwargs) - - +# TODO: Figure out wtf to do with this class class FifoQueue(object): """An abstraction class implemented for a simple push/pull queue. @@ -54,14 +22,16 @@ class FifoQueue(object): def __len__(self): """Return the length of the Redis list.""" - return redis.llen(self._queue_name) + #return redis.llen(self._queue_name) + return 0 def push(self, value): """Push a value onto the right side of the queue.""" - redis.rpush(self._queue_name, json.dumps(value)) + #redis.rpush(self._queue_name, json.dumps(value)) def pop(self): """Retrieve a value from the left side of the queue.""" - answer = redis.lpop(self._queue_name) + #answer = redis.lpop(self._queue_name) + answer = None if answer: return json.loads(answer) diff --git a/awx/settings/defaults.py b/awx/settings/defaults.py index d17368649a..18e44cdc72 100644 --- a/awx/settings/defaults.py +++ b/awx/settings/defaults.py @@ -1015,6 +1015,11 @@ LOGGING = { 'level': 'WARNING', 'propagate': False, }, + 'qpid.messaging': { + 'handlers': ['console', 'file', 'tower_warnings'], + 'propagate': False, + 'level': 'WARNING', + }, 'py.warnings': { 'handlers': ['console'], }, diff --git a/awx/settings/local_settings.py.docker_compose b/awx/settings/local_settings.py.docker_compose index 2e27a664ea..e4b47f2ebc 100644 --- a/awx/settings/local_settings.py.docker_compose +++ b/awx/settings/local_settings.py.docker_compose @@ -48,8 +48,23 @@ if is_testing(sys.argv): MONGO_DB = 'system_tracking_test' +# Django Caching Configuration +if is_testing(): + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + }, + } +else: + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': 'memcached:11211', + }, + } + # Celery AMQP configuration. -BROKER_URL = 'redis://redis/' +BROKER_URL = 'qpid://qpid:5672' # Mongo host configuration MONGO_HOST = NotImplemented diff --git a/requirements/requirements.txt b/requirements/requirements.txt index fcc2c66acc..ea7376531e 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -21,7 +21,6 @@ django-extensions==1.5.9 git+https://github.com/chrismeyersfsu/django-jsonbfield@fix-sqlite_serialization#egg=jsonbfield django-polymorphic==0.7.2 django-radius==1.0.0 -django-redis-cache==1.6.5 djangorestframework==3.3.2 djangorestframework-yaml==1.0.2 django-split-settings==0.1.1 @@ -96,6 +95,7 @@ python-heatclient==0.8.1 python-ironicclient==1.0.0 python-keystoneclient==2.1.1 python-ldap==2.4.20 +python-memcached==1.58 python-neutronclient==4.0.0 python-novaclient==3.2.0 python-openid==2.2.5 @@ -108,11 +108,11 @@ python-troveclient==1.4.0 pytz==2015.7 PyYAML==3.11 pyzmq==14.5.0 +qpid-python==0.32.1 rackspace-auth-openstack==1.3 rackspace-novaclient==1.5 rax-default-network-flags-python-novaclient-ext==0.3.2 rax-scheduled-images-python-novaclient-ext==0.3.1 -redis==2.10.3 requests-oauthlib==0.5.0 requests==2.9.1 requestsexceptions==1.1.1 diff --git a/requirements/requirements_dev.txt b/requirements/requirements_dev.txt index d7906ce28f..d848638306 100644 --- a/requirements/requirements_dev.txt +++ b/requirements/requirements_dev.txt @@ -10,3 +10,4 @@ pytest-cov pytest-django pytest-pythonpath pytest-mock +qpid-tools diff --git a/tools/docker-compose.yml b/tools/docker-compose.yml index 69a9393ed3..4b79997915 100644 --- a/tools/docker-compose.yml +++ b/tools/docker-compose.yml @@ -2,13 +2,16 @@ version: '2' services: # Primary Tower Development Container tower: - build: ./docker-compose + build: + context: .. + dockerfile: tools/docker-compose/Dockerfile ports: - "8080:8080" - "8013:8013" links: - postgres - - redis + - memcached + - qpid # - sync # volumes_from: # - sync @@ -19,9 +22,11 @@ services: postgres: image: postgres:9.4.1 - # Redis Container - redis: - image: redis:3.0.1 + memcached: + image: memcached:alpine + + qpid: + image: fedora/qpid:latest # Source Code Synchronization Container # sync: diff --git a/tools/docker-compose/Dockerfile b/tools/docker-compose/Dockerfile index 9216ae90a9..fa25fcc018 100644 --- a/tools/docker-compose/Dockerfile +++ b/tools/docker-compose/Dockerfile @@ -1,30 +1,31 @@ -FROM ubuntu:14.04 +FROM centos:7 -RUN locale-gen en_US.UTF-8 -ENV LANG en_US.UTF-8 -ENV LANGUAGE en_US:en -ENV LC_ALL en_US.UTF-8 -RUN apt-get update && apt-get install -y software-properties-common python-software-properties curl -RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository -y ppa:chris-lea/node.js; add-apt-repository -y ppa:ansible/ansible; add-apt-repository -y ppa:jal233/proot; -RUN curl -sL https://deb.nodesource.com/setup_6.x | bash - -RUN curl -sL https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - -RUN echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list -RUN apt-get update && apt-get install -y openssh-server ansible mg vim tmux git mercurial subversion python-dev python-psycopg2 make postgresql-client libpq-dev nodejs python-psutil libxml2-dev libxslt-dev lib32z1-dev libsasl2-dev libldap2-dev libffi-dev libzmq-dev proot python-pip libxmlsec1-dev swig redis-server libgss-dev libkrb5-dev && apt-get autoremove --purge -y && rm -rf /var/lib/apt/lists/* +# RUN locale-gen en_US.UTF-8 +# ENV LANG en_US.UTF-8 +# ENV LANGUAGE en_US:en +# ENV LC_ALL en_US.UTF-8 +ADD Makefile /tmp/Makefile +RUN mkdir /tmp/requirements +ADD requirements/requirements.txt requirements/requirements_ansible.txt requirements/requirements_dev.txt requirements/requirements_jenkins.txt /tmp/requirements/ +RUN yum -y update && yum -y install curl epel-release +RUN curl --silent --location https://rpm.nodesource.com/setup_6.x | bash - +RUN yum -y localinstall http://yum.postgresql.org/9.4/redhat/rhel-6-x86_64/pgdg-centos94-9.4-1.noarch.rpm +ADD tools/docker-compose/proot.repo /etc/yum.repos.d/proot.repo +RUN yum -y update && yum -y install openssh-server ansible mg vim tmux git mercurial subversion python-devel python-psycopg2 make postgresql postgresql-devel nodejs python-psutil libxml2-devel libxslt-devel libstdc++.so.6 gcc cyrus-sasl-devel cyrus-sasl openldap-devel libffi-devel zeromq-devel proot python-pip xmlsec1-devel swig krb5-devel xmlsec1-openssl xmlsec1 xmlsec1-openssl-devel libtool-ltdl-devel RUN pip install flake8 pytest pytest-pythonpath pytest-django pytest-cov pytest-mock dateutils django-debug-toolbar==1.4 pyflakes==1.0.0 virtualenv RUN /usr/bin/ssh-keygen -q -t rsa -N "" -f /root/.ssh/id_rsa RUN mkdir -p /etc/tower RUN mkdir -p /data/db -ADD license /etc/awx/license -ADD license /etc/tower/license +ADD tools/docker-compose/license /etc/tower/license RUN pip2 install honcho -RUN wget https://github.com/Yelp/dumb-init/releases/download/v1.0.1/dumb-init_1.0.1_amd64.deb -RUN dpkg -i dumb-init_*.deb -RUN rm -rf /dumb-init_*.deb -ADD ansible-tower.egg-link /tmp/ansible-tower.egg-link -ADD tower-manage /usr/local/bin/tower-manage -ADD awx-manage /usr/local/bin/awx-manage -ADD ansible_tower.egg-info /tmp/ansible_tower.egg-info +RUN curl -LO https://github.com/Yelp/dumb-init/releases/download/v1.1.3/dumb-init_1.1.3_amd64 && chmod +x ./dumb-init_1.1.3_amd64 && mv ./dumb-init_1.1.3_amd64 /usr/bin/dumb-init +ADD tools/docker-compose/ansible-tower.egg-link /tmp/ansible-tower.egg-link +ADD tools/docker-compose/tower-manage /usr/local/bin/tower-manage +ADD tools/docker-compose/awx-manage /usr/local/bin/awx-manage +ADD tools/docker-compose/ansible_tower.egg-info /tmp/ansible_tower.egg-info RUN ln -s /tower_devel/tools/docker-compose/start_development.sh /start_development.sh +WORKDIR /tmp +RUN SWIG_FEATURES="-cpperraswarn -includeall -D__`uname -m`__ -I/usr/include/openssl" VENV_BASE="/venv" make requirements_dev EXPOSE 8013 8080 22 ENTRYPOINT ["/usr/bin/dumb-init"] diff --git a/tools/docker-compose/awx-manage b/tools/docker-compose/awx-manage index e768719d3f..5d1a95e816 100755 --- a/tools/docker-compose/awx-manage +++ b/tools/docker-compose/awx-manage @@ -1,4 +1,4 @@ -#!/tower_devel/venv/tower/bin/python +#!/venv/tower/bin/python # EASY-INSTALL-ENTRY-SCRIPT: 'ansible-tower==3.0.0-0.devel','console_scripts','awx-manage' __requires__ = 'ansible-tower==3.0.0-0.devel' import sys diff --git a/tools/docker-compose/proot.repo b/tools/docker-compose/proot.repo new file mode 100644 index 0000000000..d62e8e5b9e --- /dev/null +++ b/tools/docker-compose/proot.repo @@ -0,0 +1,8 @@ +[jlaska-proot] +name=Copr repo for proot owned by jlaska +baseurl=https://copr-be.cloud.fedoraproject.org/results/jlaska/proot/epel-7-$basearch/ +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/jlaska/proot/pubkey.gpg +enabled=1 +enabled_metadata=1 diff --git a/tools/docker-compose/start_development.sh b/tools/docker-compose/start_development.sh index 16d859a3ce..b75ef757c1 100755 --- a/tools/docker-compose/start_development.sh +++ b/tools/docker-compose/start_development.sh @@ -3,10 +3,13 @@ set +x # Wait for the databases to come up ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=postgres port=5432" all -ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=redis port=6379" all +ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=memcached port=11211" all +ansible -i "127.0.0.1," -c local -v -m wait_for -a "host=qpid port=5672" all # In case Tower in the container wants to connect to itself, use "docker exec" to attach to the container otherwise -/etc/init.d/ssh start +# TODO: FIX +#/etc/init.d/ssh start + ansible -i "127.0.0.1," -c local -v -m postgresql_user -U postgres -a "name=awx-dev password=AWXsome1 login_user=postgres login_host=postgres" all ansible -i "127.0.0.1," -c local -v -m postgresql_db -U postgres -a "name=awx-dev owner=awx-dev login_user=postgres login_host=postgres" all @@ -25,11 +28,11 @@ cp -nR /tmp/ansible_tower.egg-info /tower_devel/ || true #if [ -f "awx/lib/.deps_built" ]; then # echo "Skipping dependency build - remove awx/lib/.deps_built to force a rebuild" #else -make requirements_dev +#SWIG_FEATURES="-cpperraswarn -includeall -D__`uname -m`__ -I/usr/include/openssl" make requirements_dev # touch awx/lib/.deps_built #fi -cp /tmp/ansible-tower.egg-link /tower_devel/venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link +cp /tmp/ansible-tower.egg-link /venv/tower/lib/python2.7/site-packages/ansible-tower.egg-link # Tower bootstrapping make version_file