From b82c0c332ec5ef165f304c85f1cca3f736318c74 Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Fri, 10 Jul 2015 12:16:35 -0400 Subject: [PATCH 1/2] Add license auditing python. --- tools/license-audit/license-audit.py | 252 +++++++++++++++++++++++++++ 1 file changed, 252 insertions(+) create mode 100755 tools/license-audit/license-audit.py diff --git a/tools/license-audit/license-audit.py b/tools/license-audit/license-audit.py new file mode 100755 index 0000000000..1b5cd0ea1c --- /dev/null +++ b/tools/license-audit/license-audit.py @@ -0,0 +1,252 @@ +#!/usr/bin/python +# +# Parse out as much licensing information as we can from our vendored directories to create a license report. +# You may need to edit this afterwords to replace any 'UNKNOWN' with actual data. + +import csv +import fnmatch +import json +import os +import re +import sys + +import yolk.pypi + +def usage(): + print "license-audit.py []" + sys.exit(1) + +def read_requirements(towerpath): + filename = '%s/awx/lib/site-packages/README' % (towerpath,) + ret = {} + f = open(filename) + if not f: + print "failed to open %s" %(filename,) + return None + data = f.readlines() + f.close() + for line in data: + if '==' in line: + m = re.match(r"(\S+)==(\S+) \((\S+)",line) + if m: + name = m.group(1) + version = m.group(2) + pathname = m.group(3) + if pathname.endswith(',') or pathname.endswith(')'): + pathname = pathname[:-1] + if pathname.endswith('/*'): + pathname = pathname[:-2] + item = {} + item['name'] = name + item['version'] = version + item['path'] = pathname + ret[name] = item + return ret + +def get_python(towerpath): + excludes = [ 'README*', '*.dist-info', 'funtests', 'easy_install.py', 'oslo', 'pkg_resources', '_markerlib' ] + directory = '%s/awx/lib/site-packages' % (towerpath,) + dirlist = os.listdir(directory) + ret = [] + for item in dirlist: + use = True + for exclude in excludes: + if fnmatch.fnmatch(item, exclude): + use = False + if use: + ret.append(item) + return ret + +def get_js(towerpath): + excludes = [ ] + directory = '%s/awx/ui/static/lib' % (towerpath,) + dirlist = os.listdir(directory) + ret = {} + for item in dirlist: + use = True + for exclude in excludes: + if fnmatch.fnmatch(item, exclude): + use = False + if use: + try: + bowerfile = open('%s/%s/bower.json' %(directory, item)) + except: + # add dummy entry (should read package.json if it exists) + pkg = {} + pkg['name'] = item + pkg['license'] = 'UNKNOWN' + pkg['url'] = 'UNKNOWN' + ret[item] = pkg + continue + pkginfo = json.load(bowerfile) + bowerfile.close() + pkg = {} + pkg['name'] = item + if pkginfo.has_key('license'): + pkg['license'] = normalize_license(pkginfo['license']) + else: + pkg['license'] = 'UNKNOWN' + if pkginfo.has_key('homepage'): + pkg['url'] = pkginfo['homepage'] + elif pkginfo.has_key('url'): + pkg['url'] = pkginfo['url'] + else: + pkg['url'] = 'UNKNOWN' + ret[item] = pkg + return ret + + +def search_requirements(requirements_dict, path): + for item in requirements_dict.values(): + if item['path'] == path: + return True + return False + +def normalize_license(license): + if not license: + return 'UNKNOWN' + license = license.replace('"','') + if license == 'None': + return 'UNKNOWN' + if license in ['Apache License, Version 2.0', 'Apache License (2.0)', 'Apache License 2.0', 'Apache-2.0', 'Apache License, v2.0']: + return 'Apache 2.0' + if license == 'ISC license': + return 'ISC' + if license == 'MIT License' or license == 'MIT license': + return 'MIT' + if license == 'BSD License' or license == 'Simplified BSD': + return 'BSD' + if license == 'LGPL': + return 'LGPL 2.1' + # Don't embed YOUR ENTIRE LICENSE in your metadata! + if license.find('Copyright 2011-2013 Jeffrey Gelens') != -1: + return 'Apache 2.0' + if license.find('https://github.com/umutbozkurt/django-rest-framework-mongoengine/blob/master/LICENSE') != -1: + return 'MIT' + if license == 'Python Software Foundation License': + return 'PSF' + return license + +def read_csv(filename): + ret = {} + f = open(filename) + if not f: + print "failed to open %s" %(filename,) + return None + reader = csv.reader(f, delimiter=',') + for line in reader: + item = {} + item['name'] = line[0] + item['license'] = line[1] + item['url'] = line[2] + item['source'] = line[3] + ret[line[0]] = item + return ret + +def write_csv(filename, data): + keys = data.keys() + keys.sort() + csvfile = open(filename, 'wb') + writer = csv.writer(csvfile, delimiter = ',', lineterminator = '\n') + for key in keys: + item = data[key] + l = (item['name'],item['license'],item['url'],item['source']) + writer.writerow(l) + csvfile.close() + + +if len(sys.argv) < 3: + usage() + +if len(sys.argv) < 4: + outputfile = sys.stdout +else: + outputfile = sys.argv[3] + +tower_path = sys.argv[1] + +# Read old license CSV +olddata = read_csv(sys.argv[2]) + +# Read python site-packages README requirements file +requirements = read_requirements(tower_path) + +if not olddata or not requirements: + print "No starting data" + sys.exit(1) + +# Get directory of vendored things from site-packages... +python_packages = get_python(tower_path) + +# ... and ensure they're noted in the requirements file +ok = True +for package in python_packages: + if not search_requirements(requirements, package): + print "%s not in requirements!" % (package,) + ok = False +if not ok: + sys.exit(1) + + +# See if there's pip things in our current license list that we don't have now +reqs = requirements.keys() +for item in olddata.values(): + if item['source'] == 'pip' and item['name'] not in reqs: + print "No longer vendoring %s" %(item['name'],) + +# Get directory of vendored JS things from the js dir +js_packages = get_js(tower_path) + +# See if there's JS things in our current license list that we don't have now +js = js_packages.keys() +for item in olddata.values(): + if item['source'] == 'js' and item['name'] not in js: + print "No longer vendoring %s" %(item['name'],) + +# Take the requirements file, and get license information where necessary +cs = yolk.pypi.CheeseShop() +for req in requirements.values(): + cs_info = cs.release_data(req['name'],req['version']) + if not cs_info: + print "Couldn't find '%s-%s'" %(req['name'],req['version']) + if not olddata.has_key(req['name']): + print "... and it's not in the current data. This needs fixed!" + sys.exit(1) + continue + license = normalize_license(cs_info['license']) + url = cs_info['home_page'] + try: + data = olddata[req['name']] + except: + print "New item %s" %(req['name']) + item = {} + item['name'] = req['name'] + item['license'] = license + item['url'] = url + item['source'] = 'pip' + olddata[req['name']] = item + continue + if license != 'UNKNOWN' and license != data['license']: + data['license'] = license + if url != 'UNKNOWN' and url != data['url']: + data['url'] = url + +# Update JS package info +for pkg in js: + if olddata.has_key(pkg): + data = olddata[pkg] + new = js_packages[pkg] + if new['license'] != 'UNKNOWN' and new['license'] != data['license']: + data['license'] = new['license'] + if new['url'] != 'UNKNOWN' and new['url'] != data['url']: + data['url'] = new['url'] + else: + item = {} + item['name'] = pkg + item['license'] = js_packages[pkg]['license'] + item['url'] = js_packages[pkg]['url'] + item['source'] = 'js' + olddata[pkg] = item + continue + +write_csv(outputfile, olddata) \ No newline at end of file From db939bc8eea500a008989eab876af12c95a2b347 Mon Sep 17 00:00:00 2001 From: Bill Nottingham Date: Fri, 10 Jul 2015 12:16:45 -0400 Subject: [PATCH 2/2] Add Tower license list. --- tools/license-audit/TowerLicenses.csv | 118 ++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 tools/license-audit/TowerLicenses.csv diff --git a/tools/license-audit/TowerLicenses.csv b/tools/license-audit/TowerLicenses.csv new file mode 100644 index 0000000000..be8843361c --- /dev/null +++ b/tools/license-audit/TowerLicenses.csv @@ -0,0 +1,118 @@ +Babel,BSD,http://babel.pocoo.org/,pip +IPy,BSD,https://github.com/autocracy/python-ipy,pip +Markdown,BSD,http://packages.python.org/Markdown/,pip +PrettyTable,BSD,http://code.google.com/p/prettytable/,pip +South,Apache 2.0,http://south.aeracode.org/,pip +amqp,LGPL 2.1,http://github.com/celery/py-amqp,pip +angular,MIT,https://github.com/angular/angular.js.git,js +angular-animate,MIT,https://github.com/angular/angular.js.git,js +angular-codemirror,MIT,https://github.com/chouseknecht/angular-codemirror,js +angular-cookies,MIT,https://github.com/angular/angular.js.git,js +angular-filters,Apache 2.0,https://github.com/frapontillo/angular-filters,js +angular-md5,MIT,https://github.com/gdi2290/angular-md5.git,js +angular-mocks,MIT,https://github.com/angular/angular.js.git,js +angular-moment,MIT,http://github.com/urish/angular-moment,js +angular-resource,MIT,https://github.com/angular/angular.js.git,js +angular-route,MIT,https://github.com/angular/angular.js.git,js +angular-sanitize,MIT,https://github.com/angular/angular.js.git,js +angular-scheduler,MIT,https://github.com/chouseknecht/angular-scheduler,js +angular-tz-extensions,MIT,https://github.com/chouseknecht/angular-tz-extensions,js +ansiconv,MIT,https://bitbucket.org/dhrrgn/ansiconv,pip +anyjson,BSD,http://bitbucket.org/runeh/anyjson/,pip +apache-libcloud,Apache 2.0,http://libcloud.apache.org/,pip +argparse,PSF,http://code.google.com/p/argparse/,pip +azure,Apache 2.0,https://github.com/WindowsAzure/azure-sdk-for-python,pip +billiard,BSD,http://github.com/celery/billiard,pip +bootstrap,MIT,https://github.com/twbs/bootstrap.git,js +bootstrap-datepicker,Apache 2.0,https://github.com/eternicode/bootstrap-datepicker,js +boto,MIT,https://github.com/boto/boto/,pip +celery,BSD,http://celeryproject.org,pip +codemirror,MIT,https://github.com/codemirror/CodeMirror.git,js +components-font-awesome,SIL Open Font License and MIT,http://fortawesome.github.io/Font-Awesome/,js +d2to1,BSD,http://pypi.python.org/pypi/d2to1,pip +d3,BSD,https://github.com/mbostock/d3.git,js +distribute,PSF or ZPL,http://packages.python.org/distribute,pip +django-auth-ldap,BSD,http://bitbucket.org/psagers/django-auth-ldap/,pip +django-celery,BSD,http://celeryproject.org,pip +django-crum,BSD,https://projects.ninemoreminutes.com/projects/django-crum/,pip +django-extensions,MIT,http://github.com/django-extensions/django-extensions,pip +django-jsonfield,BSD,http://bitbucket.org/schinckel/django-jsonfield/,pip +django-qsstats-magic,MIT,http://bitbucket.org/kmike/django-qsstats-magic/,pip +django-rest-framework-mongoengine,MIT,https://github.com/umutbozkurt/django-rest-framework-mongoengine,pip +django-split-settings,BSD,http://github.com/2general/django-split-settings,pip +django-taggit,BSD,http://github.com/alex/django-taggit/tree/master,pip +django_polymorphic,BSD,https://github.com/chrisglass/django_polymorphic,pip +djangorestframework,BSD,http://www.django-rest-framework.org,pip +dogpile.cache,BSD,http://bitbucket.org/zzzeek/dogpile.cache,pip +dogpile.core,BSD,http://bitbucket.org/zzzeek/dogpile.core,pip +ember-cli-test-loader,MIT,https://github.com/rjackson/ember-cli-test-loader,js +gevent-socketio,BSD,https://github.com/abourget/gevent-socketio,pip +gevent-websocket,Apache 2.0,https://bitbucket.org/Jeffrey/gevent-websocket,pip +httplib2,MIT,https://github.com/jcgregorio/httplib2,pip +importlib,PSF,https://pypi.python.org/pypi/importlib,pip +iso8601,MIT,https://bitbucket.org/micktwomey/pyiso8601,pip +isodate,BSD,http://cheeseshop.python.org/pypi/isodate,pip +jQuery.dotdotdot,MIT and GPL (*Ansible licenses via MIT),https://github.com/BeSite/jQuery.dotdotdot,js +javascript-detect-element-resize,MIT,https://github.com/sdecima/javascript-detect-element-resize,js +jquery,MIT,https://github.com/jquery/jquery,js +jquery-ui,MIT,http://jqueryui.com/,js +jqueryui,MIT,http://jqueryui.com/,js +js-yaml,MIT,https://github.com/nodeca/js-yaml,js +jsonlint,MIT,https://github.com/zaach/jsonlint.git,js +kapusta-jquery.sparkline,BSD,http://omnipotent.net/jquery.sparkline/,js +keyring,PSF and MIT,http://bitbucket.org/kang/python-keyring-lib,pip +kombu,BSD,http://kombu.readthedocs.org,pip +loader.js,MIT,https://github.com/stefanpenner/loader.js,js +lodash,MIT,https://github.com/lodash/lodash,js +lrInfiniteScroll,MIT,https://github.com/lorenzofox3/lrInfiniteScroll,js +mock,BSD,http://www.voidspace.org.uk/python/mock/,pip +moment,MIT,http://momentjs.com/,js +mongoengine,MIT,http://mongoengine.org/,pip +netaddr,BSD,https://github.com/drkjam/netaddr/,pip +nvd3,Apache 2.0,http://www.nvd3.org,js +ordereddict,MIT,https://pypi.python.org/pypi/ordereddict,pip +os-client-config,Apache 2.0,http://www.openstack.org/,pip +os_diskconfig_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_diskconfig_python_novaclient_ext,pip +os_networksv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_networksv2_python_novaclient_ext,pip +os_virtual_interfacesv2_python_novaclient_ext,Apache 2.0,https://github.com/rackerlabs/os_virtual_interfacesv2_ext,pip +oslo.config,Apache 2.0,https://launchpad.net/oslo,pip +oslo.i18n,Apache 2.0,http://launchpad.net/oslo,pip +oslo.serialization,Apache 2.0,http://launchpad.net/oslo,pip +oslo.utils,Apache 2.0,http://launchpad.net/oslo,pip +pbr,Apache 2.0,http://pypi.python.org/pypi/pbr,pip +pexpect,ISC,http://pexpect.readthedocs.org/,pip +pip,MIT,http://www.pip-installer.org,pip +psphere,Apache 2.0,https://github.com/jkinred/psphere,pip +pyrax,Apache 2.0,https://github.com/rackspace/pyrax,pip +python-cinderclient,Apache 2.0,http://www.openstack.org/,pip +python-dateutil,BSD,https://dateutil.readthedocs.org,pip +python-glanceclient,Apache 2.0,http://www.openstack.org/,pip +python-ironicclient,Apache 2.0,http://www.openstack.org/,pip +python-keystoneclient,Apache 2.0,http://www.openstack.org/,pip +python-neutronclient,Apache 2.0,http://www.openstack.org/,pip +python-novaclient,Apache 2.0,https://git.openstack.org/cgit/openstack/python-novaclient,pip +python-swiftclient,Apache 2.0,http://www.openstack.org/,pip +python-troveclient,Apache 2.0,http://www.openstack.org/,pip +pytz,MIT,http://pythonhosted.org/pytz,pip +pywinrm,MIT,http://github.com/diyan/pywinrm/,pip +rackspace-auth-openstack,Apache 2.0,https://github.com/rackerlabs/rackspace-auth-openstack,pip +rackspace-novaclient,Apache 2.0,https://github.com/rackerlabs/rackspace-novaclient,pip +rax_default_network_flags_python_novaclient_ext,Apache 2.0,https://github.com/rackspace/rax_default_network_flags_python_novaclient_ext,pip +rax_scheduled_images_python_novaclient_ext,Apache 2.0,https://github.com/rackspace-titan/rax_scheduled_images_python_novaclient_ext,pip +redis,MIT,http://github.com/andymccurdy/redis-py,pip +requests,Apache 2.0,http://python-requests.org,pip +rrule,BSD,https://github.com/jakubroztocil/rrule.git,js +scrollto,MIT,https://github.com/balupton/jquery-scrollto,js +select2,MIT,https://github.com/chrisjbaik/select2.git,js +setuptools,PSF or ZPL,https://bitbucket.org/pypa/setuptools,pip +shade,Apache 2.0,http://ci.openstack.org/,pip +simplejson,MIT,http://github.com/simplejson/simplejson,pip +six,MIT,http://pypi.python.org/pypi/six/,pip +sizzle,MIT,https://github.com/jquery/sizzle.git,js +socket.io-client,MIT,https://github.com/Automattic/socket.io-client.git,js +stevedore,Apache 2.0,https://github.com/dreamhost/stevedore,pip +suds,LGPL 3,https://fedorahosted.org/suds,pip +timezone-js,Apache 2.0,https://github.com/mde/timezone-js.git,js +twitter,MIT,http://getbootstrap.com,js +underscore,MIT,https://github.com/jashkenas/underscore,js +xmltodict,MIT,https://github.com/martinblech/xmltodict,pip