Merge remote-tracking branch 'upstream/release_2.3' into devel

* upstream/release_2.3: (91 commits)
  Include python-{paramiko,ecdsa} dependencies
  Remove extra epel testing stanzas
  Unit test for ec2 credentialless inventory
  Fix issue with ec2 iam sync with no credential.
  Use the htpasswd command instead the ansible module
  Pip is no longer needed
  check local user root or not in ./configure
  Remove unneeded when check for super user addition
  Improve distro detection in setup.sh
  Fix superuser check on upgrade
  Minor improvements to setup.sh
  Remove ansible prerequisite check from configure
  Attempt to install ansible within setup.sh
  Allow munin processes to access postgres
  Move up base package dependency install
  fixes jenkins failures
  Proper flake8 fix
  fixes executing processes with correct PYTHONPATH will pickup .pth files
  Show the repo for bundled package file dump
  Proper flake8 fix
  ...

tools/docker/Dockerfile

@@ -6,7 +6,7 @@ ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8  
 RUN apt-get update
 RUN apt-get install -y software-properties-common python-software-properties curl
-RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository ppa:ansible/ansible
+RUN add-apt-repository -y ppa:chris-lea/redis-server; add-apt-repository -y ppa:chris-lea/zeromq; add-apt-repository -y ppa:chris-lea/node.js; add-apt-repository ppa:ansible/ansible
 RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash -
 RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10 && apt-key adv --fetch-keys http://www.postgresql.org/media/keys/ACCC4CF8.asc
 RUN echo "deb http://repo.mongodb.org/apt/ubuntu "$(lsb_release -sc)"/mongodb-org/3.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.0.list && echo "deb http://apt.postgresql.org/pub/repos/apt/ trusty-pgdg main" | tee /etc/apt/sources.list.d/postgres-9.4.list

tools/license-audit/TowerLicenses.csv

@@ -1,7 +1,8 @@
 Babel,BSD,http://babel.pocoo.org/,pip
-IPy,BSD,https://github.com/autocracy/python-ipy,pip
+Django,BSD,http://www.djangoproject.com/,pip
 Markdown,BSD,http://packages.python.org/Markdown/,pip
 PrettyTable,BSD,http://code.google.com/p/prettytable/,pip
+PyYAML,MIT,http://pyyaml.org/wiki/PyYAML,pip
 South,Apache 2.0,http://south.aeracode.org/,pip
 amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip
 angular,MIT,https://github.com/angular/angular.js.git,js
@@ -27,8 +28,12 @@ bootstrap,MIT,https://github.com/twbs/bootstrap.git,js
 bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js
 boto,MIT,https://github.com/boto/boto/,pip
 celery,BSD,http://celeryproject.org,pip
+cffi,MIT,http://cffi.readthedocs.org,pip
+cliff,Apache 2.0,https://launchpad.net/python-cliff,pip
+cmd2,MIT,http://packages.python.org/cmd2/,pip
 codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js
 components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js
+cryptography,BSD or Apache 2.0,https://github.com/pyca/cryptography,pip
 d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip
 d3,BSD,https://github.com/mbostock/d3.git,js
 distribute,PSF or ZPL,http://packages.python.org/distribute,pip
@@ -37,6 +42,7 @@ django-celery,BSD,http://celeryproject.org,pip
 django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip
 django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip
 django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip
+django-polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip
 django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip
 django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip
 django-split-settings,BSD,http://github.com/2general/django-split-settings,pip
@@ -46,10 +52,16 @@ djangorestframework,BSD,http://www.django-rest-framework.org,pip
 dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip
 dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip
 ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js
+enum34,BSD,https://pypi.python.org/pypi/enum34,pip
+gevent,MIT,http://www.gevent.org/,pip
 gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip
 gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip
+greenlet,MIT,https://github.com/python-greenlet/greenlet,pip
 httplib2,MIT,https://github.com/jcgregorio/httplib2,pip
+idna,BSD,https://github.com/kjd/idna,pip
 importlib,PSF,https://pypi.python.org/pypi/importlib,pip
+ipaddress,PSF,https://github.com/phihag/ipaddress,pip
+ipy,BSD,https://github.com/autocracy/python-ipy,pip
 iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip
 isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip
 jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js
@@ -59,19 +71,28 @@ jquery-ui,MIT,http://jqueryui.com/,js
 jqueryui,MIT,http://jqueryui.com/,js
 js-yaml,MIT,https://github.com/nodeca/js-yaml,js
 jsonlint,MIT,https://github.com/zaach/jsonlint.git,js
+jsonpatch,BSD,https://github.com/stefankoegl/python-json-patch,pip
+jsonpointer,BSD,https://github.com/stefankoegl/python-json-pointer,pip
+jsonschema,MIT,http://github.com/Julian/jsonschema,pip
 kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js
 keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip
 kombu,BSD,http://kombu.readthedocs.org,pip
 loader.js,MIT,https://github.com/stefanpenner/loader.js,js
 lodash,MIT,https://github.com/lodash/lodash,js
 lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js
+lxml,BSD,http://lxml.de/,pip
 mock,BSD,http://www.voidspace.org.uk/python/mock/,pip
 moment,MIT,http://momentjs.com/,js
 mongoengine,MIT,http://mongoengine.org/,pip
+msgpack-python,Apache 2.0,http://msgpack.org/,pip
 netaddr,BSD,https://github.com/drkjam/netaddr/,pip
+netifaces,MIT,https://bitbucket.org/al45tair/netifaces,pip
 nvd3,Apache 2.0,http://www.nvd3.org,js
 ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip
 os-client-config,Apache 2.0,http://www.openstack.org/,pip
+os-diskconfig-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
+os-networksv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
+os-virtual-interfacesv2-python-novaclient-ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
 os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip
 os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip
 os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip
@@ -82,19 +103,29 @@ oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip
 pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip
 pexpect,ISC,http://pexpect.readthedocs.org/,pip
 pip,MIT,http://www.pip-installer.org,pip
+prettytable,BSD,http://code.google.com/p/prettytable/,pip
 psphere,Apache 2.0,https://github.com/jkinred/psphere,pip
+psycopg2,LGPL with exceptions or ZPL,http://initd.org/psycopg/,pip
+pyOpenSSL,Apache 2.0,https://github.com/pyca/pyopenssl,pip
+pyasn1,BSD,http://sourceforge.net/projects/pyasn1/,pip
+pycparser,BSD,https://github.com/eliben/pycparser,pip
+pycrypto,Public domain,http://www.pycrypto.org/,pip
+pymongo,Apache 2.0,http://github.com/mongodb/mongo-python-driver,pip
+pyparsing,MIT,http://pyparsing.wikispaces.com/,pip
 pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip
 python-cinderclient,Apache 2.0,http://www.openstack.org/,pip
 python-dateutil,BSD,https://dateutil.readthedocs.org,pip
 python-glanceclient,Apache 2.0,http://www.openstack.org/,pip
 python-ironicclient,Apache 2.0,http://www.openstack.org/,pip
 python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip
+python-ldap,Python style,http://www.python-ldap.org/,pip
 python-neutronclient,Apache 2.0,http://www.openstack.org/,pip
 python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip
 python-swiftclient,Apache 2.0,http://www.openstack.org/,pip
 python-troveclient,Apache 2.0,http://www.openstack.org/,pip
 pytz,MIT,http://pythonhosted.org/pytz,pip
 pywinrm,MIT,http://github.com/diyan/pywinrm/,pip
+pyzmq,LGPL+BSD,http://github.com/zeromq/pyzmq,pip
 rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip
 rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip
 rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/rax_default_network_flags_python_novaclient_ext,pip
@@ -112,7 +143,11 @@ sizzle,MIT,https://github.com/jquery/sizzle.git,js
 socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js
 stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip
 suds,LGPL 3,https://fedorahosted.org/suds,pip
+superlance,BSD,http://supervisord.org,pip
 timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js
 twitter,MIT,http://getbootstrap.com,js
 underscore,MIT,https://github.com/jashkenas/underscore,js
+warlock,Apache 2.0,http://github.com/bcwaldon/warlock,pip
+wheel,MIT,http://bitbucket.org/pypa/wheel/,pip
+wsgiref,PSF or ZPL,http://cheeseshop.python.org/pypi/wsgiref,pip
 xmltodict,MIT,https://github.com/martinblech/xmltodict,pip

tools/license-audit/license-audit.py

@@ -17,52 +17,55 @@ def usage():
     sys.exit(1)
 
 def read_requirements(towerpath):
-    filename = '%s/awx/lib/site-packages/README' % (towerpath,)
+    filename = '%s/requirements/requirements.txt' % (towerpath,)
     ret = {}
-    f = open(filename)
-    if not f:
+    try:
+        f = open(filename)
+    except:
         print "failed to open %s" %(filename,)
         return None
     data = f.readlines()
     f.close()
     for line in data:
+        if line[0] == '#':
+            continue
         if '==' in line:
-            m = re.match(r"(\S+)==(\S+) \((\S+)",line)
+            m = re.match(r"(\S+)==(\S+)",line)
             if m:
                 name = m.group(1)
                 version = m.group(2)
-                pathname = m.group(3)
-                if pathname.endswith(',') or pathname.endswith(')'):
-                    pathname = pathname[:-1]
-                if pathname.endswith('/*'):
-                    pathname = pathname[:-2]
                 item = {}
                 item['name'] = name
                 item['version'] = version
-                item['path'] = pathname
                 ret[name] = item
-    return ret
-
-def get_python(towerpath):
-    excludes = [ 
-        'README*', 
-        '*.dist-info', 
-        'funtests', 
-        'easy_install.py', 
-        'oslo', 
-        'pkg_resources', 
-        '_markerlib'
-    ]
-    directory = '%s/awx/lib/site-packages' % (towerpath,)
-    dirlist = os.listdir(directory)
-    ret = []
-    for item in dirlist:
-        use = True
-        for exclude in excludes:
-            if fnmatch.fnmatch(item, exclude):
-                use = False
-        if use:
-            ret.append(item)
+                continue
+        elif line.startswith("git+https"):
+            l = line.rsplit('/',1)
+            m = re.match(r"(\S+).git@(\S+)#",l[1])
+            if m:
+                name = m.group(1)
+                version = m.group(2)
+                if version.startswith('tower_'):
+                    version = version[6:]
+                item = {}
+                if name == 'python-ipy':
+                    name='ipy'
+                item['name'] = name
+                item['version'] = version
+                if len(version) > 20:
+                    # it's a sha1sum, read it off the egg spec
+                    lver = l[1].rsplit('-',1)
+                    if lver[1] == l[1]:
+                        lver = l[1].rsplit('_',1)
+                    item['version'] = lver[1][:-1]
+                ret[name] = item
+                continue
+        else:
+            item = {}
+            item['name'] = line[:-1]
+            item['version'] = ''
+            ret[name] = item
+            continue
     return ret
 
 def get_js(towerpath):
@@ -116,13 +119,13 @@ def normalize_license(license):
     license = license.replace('"','')
     if license == 'None':
         return 'UNKNOWN'
-    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']:
+    if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0', 'APL2']:
         return 'Apache 2.0'
     if license == 'ISC license':
         return 'ISC'
     if license == 'MIT License' or license == 'MIT license':
         return 'MIT'
-    if license == 'BSD License' or license == 'Simplified BSD':
+    if license in ['BSD License', 'Simplified BSD', 'BSD-derived (http://www.repoze.org/LICENSE.txt)', 'BSD-like', 'Modified BSD License']:
         return 'BSD'
     if license == 'LGPL':
         return 'LGPL 2.1'
@@ -131,6 +134,10 @@ def normalize_license(license):
         return 'Apache 2.0'
     if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1:
         return 'MIT'
+    if license == '"BSD or Apache License, Version 2.0"':
+        return 'BSD or Apache 2.0'
+    if license == 'Modified BSD License':
+        return 'BSD'
     if license == 'Python Software Foundation License':
         return 'PSF'
     return license
@@ -183,24 +190,11 @@ if not olddata or not requirements:
     print "No starting data"
     sys.exit(1)
 
-# Get directory of vendored things from site-packages...
-python_packages = get_python(tower_path)
-
-# ... and ensure they're noted in the requirements file
-ok = True
-for package in python_packages:
-    if not search_requirements(requirements, package):
-        print "%s not in requirements!" % (package,)
-        ok = False
-if not ok:
-    sys.exit(1)
-
-
 # See if there's pip things in our current license list that we don't have now
 reqs = requirements.keys()
 for item in olddata.values():
     if item['source'] == 'pip' and item['name'] not in reqs:
-        print "No longer vendoring %s" %(item['name'],)
+        print "Potentially no longer vendoring %s" %(item['name'],)
 
 # Get directory of vendored JS things from the js dir
 js_packages = get_js(tower_path)
@@ -214,10 +208,12 @@ for item in olddata.values():
 # Take the requirements file, and get license information where necessary
 cs = yolk.pypi.CheeseShop()
 for req in requirements.values():
-    cs_info = cs.release_data(req['name'],req['version'])
+    # name sanitization
+    (pname, pvers) = cs.query_versions_pypi(req['name'])
+    cs_info = cs.release_data(pname,req['version'])
     if not cs_info:
-        print "Couldn't find '%s-%s'" %(req['name'],req['version'])
-        if 'name' not in olddata:
+        print "Couldn't find '%s==%s'" %(req['name'],req['version'])
+        if req['name'] not in olddata:
             print "... and it's not in the current data. This needs fixed!"
             sys.exit(1)
         continue
@@ -241,7 +237,7 @@ for req in requirements.values():
 
 # Update JS package info
 for pkg in js:
-    if 'pkg' in olddata:
+    if pkg in olddata:
         data = olddata[pkg]
         new = js_packages[pkg]
         if new['license'] != 'UNKNOWN' and new['license'] != data['license']: