diff --git a/awx/lib/site-packages/README b/awx/lib/site-packages/README index 7d8db8e75b..544fddd707 100644 --- a/awx/lib/site-packages/README +++ b/awx/lib/site-packages/README @@ -38,7 +38,7 @@ ordereddict==1.1 (ordereddict.py, needed for Python 2.6 support) os-diskconfig-python-novaclient-ext==0.1.2 (os_diskconfig_python_novaclient_ext/*) os-networksv2-python-novaclient-ext==0.21 (os_networksv2_python_novaclient_ext.py) os-virtual-interfacesv2-python-novaclient-ext==0.15 (os_virtual_interfacesv2_python_novaclient_ext.py) -pbr==0.8.0 (pbr/*) +pbr==0.10.0 (pbr/*) pexpect==3.3 (pexpect/*, excluded pxssh.py, fdpexpect.py, FSM.py, screen.py, ANSI.py) pip==1.5.4 (pip/*, excluded bin/pip*) diff --git a/awx/lib/site-packages/pbr/core.py b/awx/lib/site-packages/pbr/core.py index f622ad000b..0f4b94a1e8 100644 --- a/awx/lib/site-packages/pbr/core.py +++ b/awx/lib/site-packages/pbr/core.py @@ -40,6 +40,7 @@ from distutils import core from distutils import errors +import logging import os import sys import warnings @@ -49,7 +50,17 @@ from setuptools import dist from pbr import util -core.Distribution = dist._get_unpatched(core.Distribution) +_saved_core_distribution = core.Distribution + + +def _monkeypatch_distribution(): + core.Distribution = dist._get_unpatched(core.Distribution) + + +def _restore_distribution_monkeypatch(): + core.Distribution = _saved_core_distribution + + if sys.version_info[0] == 3: string_type = str integer_types = (int,) @@ -76,52 +87,63 @@ def pbr(dist, attr, value): not work well with distributions that do use a `Distribution` subclass. """ - if not value: - return - if isinstance(value, string_type): - path = os.path.abspath(value) - else: - path = os.path.abspath('setup.cfg') - if not os.path.exists(path): - raise errors.DistutilsFileError( - 'The setup.cfg file %s does not exist.' % path) - - # Converts the setup.cfg file to setup() arguments try: - attrs = util.cfg_to_args(path) - except Exception: - e = sys.exc_info()[1] - raise errors.DistutilsSetupError( - 'Error parsing %s: %s: %s' % (path, e.__class__.__name__, e)) + _monkeypatch_distribution() + if not value: + return + if isinstance(value, string_type): + path = os.path.abspath(value) + else: + path = os.path.abspath('setup.cfg') + if not os.path.exists(path): + raise errors.DistutilsFileError( + 'The setup.cfg file %s does not exist.' % path) - # Repeat some of the Distribution initialization code with the newly - # provided attrs - if attrs: - # Skips 'options' and 'licence' support which are rarely used; may add - # back in later if demanded - for key, val in attrs.items(): - if hasattr(dist.metadata, 'set_' + key): - getattr(dist.metadata, 'set_' + key)(val) - elif hasattr(dist.metadata, key): - setattr(dist.metadata, key, val) - elif hasattr(dist, key): - setattr(dist, key, val) - else: - msg = 'Unknown distribution option: %s' % repr(key) - warnings.warn(msg) + # Converts the setup.cfg file to setup() arguments + try: + attrs = util.cfg_to_args(path) + except Exception: + e = sys.exc_info()[1] + # NB: This will output to the console if no explicit logging has + # been setup - but thats fine, this is a fatal distutils error, so + # being pretty isn't the #1 goal.. being diagnosable is. + logging.exception('Error parsing') + raise errors.DistutilsSetupError( + 'Error parsing %s: %s: %s' % (path, e.__class__.__name__, e)) - # Re-finalize the underlying Distribution - core.Distribution.finalize_options(dist) + # Repeat some of the Distribution initialization code with the newly + # provided attrs + if attrs: + # Skips 'options' and 'licence' support which are rarely used; may + # add back in later if demanded + for key, val in attrs.items(): + if hasattr(dist.metadata, 'set_' + key): + getattr(dist.metadata, 'set_' + key)(val) + elif hasattr(dist.metadata, key): + setattr(dist.metadata, key, val) + elif hasattr(dist, key): + setattr(dist, key, val) + else: + msg = 'Unknown distribution option: %s' % repr(key) + warnings.warn(msg) - # This bit comes out of distribute/setuptools - if isinstance(dist.metadata.version, integer_types + (float,)): - # Some people apparently take "version number" too literally :) - dist.metadata.version = str(dist.metadata.version) + # Re-finalize the underlying Distribution + core.Distribution.finalize_options(dist) - # This bit of hackery is necessary so that the Distribution will ignore - # normally unsupport command options (namely pre-hooks and post-hooks). - # dist.command_options is normally a dict mapping command names to dicts of - # their options. Now it will be a defaultdict that returns IgnoreDicts for - # the each command's options so we can pass through the unsupported options - ignore = ['pre_hook.*', 'post_hook.*'] - dist.command_options = util.DefaultGetDict(lambda: util.IgnoreDict(ignore)) + # This bit comes out of distribute/setuptools + if isinstance(dist.metadata.version, integer_types + (float,)): + # Some people apparently take "version number" too literally :) + dist.metadata.version = str(dist.metadata.version) + + # This bit of hackery is necessary so that the Distribution will ignore + # normally unsupport command options (namely pre-hooks and post-hooks). + # dist.command_options is normally a dict mapping command names to + # dicts of their options. Now it will be a defaultdict that returns + # IgnoreDicts for the each command's options so we can pass through the + # unsupported options + ignore = ['pre_hook.*', 'post_hook.*'] + dist.command_options = util.DefaultGetDict( + lambda: util.IgnoreDict(ignore) + ) + finally: + _restore_distribution_monkeypatch() diff --git a/awx/lib/site-packages/pbr/packaging.py b/awx/lib/site-packages/pbr/packaging.py index 4d1332ed03..471ed85683 100644 --- a/awx/lib/site-packages/pbr/packaging.py +++ b/awx/lib/site-packages/pbr/packaging.py @@ -49,9 +49,6 @@ from pbr import extra_files TRUE_VALUES = ('true', '1', 'yes') REQUIREMENTS_FILES = ('requirements.txt', 'tools/pip-requires') TEST_REQUIREMENTS_FILES = ('test-requirements.txt', 'tools/test-requires') -# part of the standard library starting with 2.7 -# adding it to the requirements list screws distro installs -BROKEN_ON_27 = ('argparse', 'importlib', 'ordereddict') def get_requirements_files(): @@ -125,6 +122,13 @@ def parse_requirements(requirements_files=None): if (not line.strip()) or line.startswith('#'): continue + # Handle nested requirements files such as: + # -r other-requirements.txt + if line.startswith('-r'): + req_file = line.partition(' ')[2] + requirements += parse_requirements([req_file]) + continue + try: project_name = pkg_resources.Requirement.parse(line).project_name except ValueError: @@ -146,10 +150,6 @@ def parse_requirements(requirements_files=None): elif re.match(r'\s*-f\s+', line): line = None reason = 'Index Location' - elif (project_name and - project_name in BROKEN_ON_27 and sys.version_info >= (2, 7)): - line = None - reason = 'Python 2.6 only dependency' if line is not None: requirements.append(line) @@ -246,95 +246,101 @@ def write_git_changelog(git_dir=None, dest_dir=os.path.curdir, """Write a changelog based on the git changelog.""" should_skip = get_boolean_option(option_dict, 'skip_changelog', 'SKIP_WRITE_GIT_CHANGELOG') - if not should_skip: - new_changelog = os.path.join(dest_dir, 'ChangeLog') - # If there's already a ChangeLog and it's not writable, just use it - if (os.path.exists(new_changelog) - and not os.access(new_changelog, os.W_OK)): - return - log.info('[pbr] Writing ChangeLog') - if git_dir is None: - git_dir = _get_git_directory() - if git_dir: - log_cmd = ['log', '--oneline', '--decorate'] - changelog = _run_git_command(log_cmd, git_dir) - first_line = True - with io.open(new_changelog, "w", - encoding="utf-8") as changelog_file: - changelog_file.write("CHANGES\n=======\n\n") - for line in changelog.split('\n'): - line_parts = line.split() - if len(line_parts) < 2: - continue - # Tags are in a list contained in ()'s. If a commit - # subject that is tagged happens to have ()'s in it - # this will fail - if line_parts[1].startswith('(') and ')' in line: - msg = line.split(')')[1].strip() - else: - msg = " ".join(line_parts[1:]) + if should_skip: + return - if "tag:" in line: - tags = [ - tag.split(",")[0] - for tag in line.split(")")[0].split("tag: ")[1:]] - tag = _get_highest_tag(tags) + new_changelog = os.path.join(dest_dir, 'ChangeLog') + # If there's already a ChangeLog and it's not writable, just use it + if (os.path.exists(new_changelog) + and not os.access(new_changelog, os.W_OK)): + return + log.info('[pbr] Writing ChangeLog') + if git_dir is None: + git_dir = _get_git_directory() + if not git_dir: + return - underline = len(tag) * '-' - if not first_line: - changelog_file.write('\n') - changelog_file.write( - ("%(tag)s\n%(underline)s\n\n" % - dict(tag=tag, - underline=underline))) + log_cmd = ['log', '--oneline', '--decorate'] + changelog = _run_git_command(log_cmd, git_dir) + first_line = True + with io.open(new_changelog, "w", + encoding="utf-8") as changelog_file: + changelog_file.write("CHANGES\n=======\n\n") + for line in changelog.split('\n'): + line_parts = line.split() + if len(line_parts) < 2: + continue + # Tags are in a list contained in ()'s. If a commit + # subject that is tagged happens to have ()'s in it + # this will fail + if line_parts[1].startswith('(') and ')' in line: + msg = line.split(')')[1].strip() + else: + msg = " ".join(line_parts[1:]) - if not msg.startswith("Merge "): - if msg.endswith("."): - msg = msg[:-1] - changelog_file.write( - ("* %(msg)s\n" % dict(msg=msg))) - first_line = False + if "tag:" in line: + tags = [ + tag.split(",")[0] + for tag in line.split(")")[0].split("tag: ")[1:]] + tag = _get_highest_tag(tags) + + underline = len(tag) * '-' + if not first_line: + changelog_file.write('\n') + changelog_file.write( + ("%(tag)s\n%(underline)s\n\n" % + dict(tag=tag, + underline=underline))) + + if not msg.startswith("Merge "): + if msg.endswith("."): + msg = msg[:-1] + changelog_file.write( + ("* %(msg)s\n" % dict(msg=msg))) + first_line = False def generate_authors(git_dir=None, dest_dir='.', option_dict=dict()): """Create AUTHORS file using git commits.""" should_skip = get_boolean_option(option_dict, 'skip_authors', 'SKIP_GENERATE_AUTHORS') - if not should_skip: - old_authors = os.path.join(dest_dir, 'AUTHORS.in') - new_authors = os.path.join(dest_dir, 'AUTHORS') - # If there's already an AUTHORS file and it's not writable, just use it - if (os.path.exists(new_authors) - and not os.access(new_authors, os.W_OK)): - return - log.info('[pbr] Generating AUTHORS') - ignore_emails = '(jenkins@review|infra@lists|jenkins@openstack)' - if git_dir is None: - git_dir = _get_git_directory() - if git_dir: - authors = [] + if should_skip: + return - # don't include jenkins email address in AUTHORS file - git_log_cmd = ['log', '--use-mailmap', '--format=%aN <%aE>'] - authors += _run_git_command(git_log_cmd, git_dir).split('\n') - authors = [a for a in authors if not re.search(ignore_emails, a)] + old_authors = os.path.join(dest_dir, 'AUTHORS.in') + new_authors = os.path.join(dest_dir, 'AUTHORS') + # If there's already an AUTHORS file and it's not writable, just use it + if (os.path.exists(new_authors) + and not os.access(new_authors, os.W_OK)): + return + log.info('[pbr] Generating AUTHORS') + ignore_emails = '(jenkins@review|infra@lists|jenkins@openstack)' + if git_dir is None: + git_dir = _get_git_directory() + if git_dir: + authors = [] - # get all co-authors from commit messages - co_authors_out = _run_git_command('log', git_dir) - co_authors = re.findall('Co-authored-by:.+', co_authors_out, - re.MULTILINE) - co_authors = [signed.split(":", 1)[1].strip() - for signed in co_authors if signed] + # don't include jenkins email address in AUTHORS file + git_log_cmd = ['log', '--format=%aN <%aE>'] + authors += _run_git_command(git_log_cmd, git_dir).split('\n') + authors = [a for a in authors if not re.search(ignore_emails, a)] - authors += co_authors - authors = sorted(set(authors)) + # get all co-authors from commit messages + co_authors_out = _run_git_command('log', git_dir) + co_authors = re.findall('Co-authored-by:.+', co_authors_out, + re.MULTILINE) + co_authors = [signed.split(":", 1)[1].strip() + for signed in co_authors if signed] - with open(new_authors, 'wb') as new_authors_fh: - if os.path.exists(old_authors): - with open(old_authors, "rb") as old_authors_fh: - new_authors_fh.write(old_authors_fh.read()) - new_authors_fh.write(('\n'.join(authors) + '\n') - .encode('utf-8')) + authors += co_authors + authors = sorted(set(authors)) + + with open(new_authors, 'wb') as new_authors_fh: + if os.path.exists(old_authors): + with open(old_authors, "rb") as old_authors_fh: + new_authors_fh.write(old_authors_fh.read()) + new_authors_fh.write(('\n'.join(authors) + '\n') + .encode('utf-8')) def _find_git_files(dirname='', git_dir=None): @@ -662,9 +668,9 @@ try: autoindex_filename = os.path.join(source_dir, 'autoindex.rst') with open(autoindex_filename, 'w') as autoindex: autoindex.write(""".. toctree:: - :maxdepth: 1 + :maxdepth: 1 - """) +""") for module in module_list: output_filename = os.path.join(source_dir, "%s.rst" % module) @@ -794,7 +800,7 @@ def _get_version_from_git(pre_version): """ git_dir = _get_git_directory() - if git_dir: + if git_dir and _git_is_installed(): if pre_version: try: return _run_git_command( diff --git a/awx/lib/site-packages/pbr/testr_command.py b/awx/lib/site-packages/pbr/testr_command.py index cf2a0750a1..34b02ed556 100644 --- a/awx/lib/site-packages/pbr/testr_command.py +++ b/awx/lib/site-packages/pbr/testr_command.py @@ -40,11 +40,14 @@ package metadata. from distutils import cmd import distutils.errors +import logging import os import sys from testrepository import commands +logger = logging.getLogger(__name__) + class Testr(cmd.Command): @@ -54,14 +57,17 @@ class Testr(cmd.Command): ('coverage', None, "Replace PYTHON with coverage and merge coverage " "from each testr worker."), ('testr-args=', 't', "Run 'testr' with these args"), - ('omit=', 'o', 'Files to omit from coverage calculations'), + ('omit=', 'o', "Files to omit from coverage calculations"), + ('coverage-package-name=', None, "Use this name for coverage package"), ('slowest', None, "Show slowest test times after tests complete."), ('no-parallel', None, "Run testr serially"), + ('log-level=', 'l', "Log level (default: info)"), ] boolean_options = ['coverage', 'slowest', 'no_parallel'] def _run_testr(self, *args): + logger.debug("_run_testr called with args = %r", args) return commands.run_argv([sys.argv[0]] + list(args), sys.stdin, sys.stdout, sys.stderr) @@ -70,18 +76,28 @@ class Testr(cmd.Command): self.coverage = None self.omit = "" self.slowest = None + self.coverage_package_name = None self.no_parallel = None + self.log_level = 'info' def finalize_options(self): + self.log_level = getattr( + logging, + self.log_level.upper(), + logging.INFO) + logging.basicConfig(level=self.log_level) + logger.debug("finalize_options called") if self.testr_args is None: self.testr_args = [] else: self.testr_args = self.testr_args.split() if self.omit: self.omit = "--omit=%s" % self.omit + logger.debug("finalize_options: self.__dict__ = %r", self.__dict__) def run(self): """Set up testr repo, then run testr""" + logger.debug("run called") if not os.path.isdir(".testrepository"): self._run_testr("init") @@ -101,12 +117,19 @@ class Testr(cmd.Command): self._coverage_after() def _coverage_before(self): + logger.debug("_coverage_before called") package = self.distribution.get_name() if package.startswith('python-'): package = package[7:] + + # Use this as coverage package name + if self.coverage_package_name: + package = self.coverage_package_name options = "--source %s --parallel-mode" % package os.environ['PYTHON'] = ("coverage run %s" % options) + logger.debug("os.environ['PYTHON'] = %r", os.environ['PYTHON']) def _coverage_after(self): + logger.debug("_coverage_after called") os.system("coverage combine") os.system("coverage html -d ./cover %s" % self.omit) diff --git a/awx/lib/site-packages/pbr/tests/base.py b/awx/lib/site-packages/pbr/tests/base.py index f18dadc2e1..ce20de1bdf 100644 --- a/awx/lib/site-packages/pbr/tests/base.py +++ b/awx/lib/site-packages/pbr/tests/base.py @@ -92,6 +92,10 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): self.log_fixture = self.useFixture( fixtures.FakeLogger('pbr')) + # Older git does not have config --local, so create a temporary home + # directory to permit using git config --global without stepping on + # developer configuration. + self.useFixture(fixtures.TempHomeDir()) self.useFixture(fixtures.NestedTempfile()) self.useFixture(fixtures.FakeLogger()) self.useFixture(fixtures.EnvironmentVariable('PBR_VERSION', '0.0')) @@ -112,17 +116,24 @@ class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): k.startswith('pbr_testpackage.')): del sys.modules[k] - def run_setup(self, *args): - return self._run_cmd(sys.executable, ('setup.py',) + args) + def run_setup(self, *args, **kwargs): + return self._run_cmd(sys.executable, ('setup.py',) + args, **kwargs) - def _run_cmd(self, cmd, args=[]): + def _run_cmd(self, cmd, args=[], allow_fail=True, cwd=None): """Run a command in the root of the test working copy. Runs a command, with the given argument list, in the root of the test working copy--returns the stdout and stderr streams and the exit code from the subprocess. + + :param cwd: If falsy run within the test package dir, otherwise run + within the named path. """ - return _run_cmd([cmd] + list(args), cwd=self.package_dir) + cwd = cwd or self.package_dir + result = _run_cmd([cmd] + list(args), cwd=cwd) + if result[2] and not allow_fail: + raise Exception("Command failed retcode=%s" % result[2]) + return result def _run_cmd(args, cwd): diff --git a/awx/lib/site-packages/pbr/tests/test_packaging.py b/awx/lib/site-packages/pbr/tests/test_packaging.py index 029f378818..46dd0d54da 100644 --- a/awx/lib/site-packages/pbr/tests/test_packaging.py +++ b/awx/lib/site-packages/pbr/tests/test_packaging.py @@ -39,6 +39,7 @@ # BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS import os +import tempfile import fixtures import mock @@ -47,18 +48,37 @@ from pbr import packaging from pbr.tests import base +class TestRepo(fixtures.Fixture): + """A git repo for testing with. + + Use of TempHomeDir with this fixture is strongly recommended as due to the + lack of config --local in older gits, it will write to the users global + configuration without TempHomeDir. + """ + + def __init__(self, basedir): + super(TestRepo, self).__init__() + self._basedir = basedir + + def setUp(self): + super(TestRepo, self).setUp() + base._run_cmd(['git', 'init', '.'], self._basedir) + base._run_cmd( + ['git', 'config', '--global', 'user.email', 'example@example.com'], + self._basedir) + base._run_cmd(['git', 'add', '.'], self._basedir) + + def commit(self): + base._run_cmd(['git', 'commit', '-m', 'test commit'], self._basedir) + + class TestPackagingInGitRepoWithCommit(base.BaseTestCase): def setUp(self): super(TestPackagingInGitRepoWithCommit, self).setUp() - self.useFixture(fixtures.TempHomeDir()) - self._run_cmd( - 'git', ['config', '--global', 'user.email', 'nobody@example.com']) - self._run_cmd('git', ['init', '.']) - self._run_cmd('git', ['add', '.']) - self._run_cmd('git', ['commit', '-m', 'test commit']) - self.run_setup('sdist') - return + repo = self.useFixture(TestRepo(self.package_dir)) + repo.commit() + self.run_setup('sdist', allow_fail=False) def test_authors(self): # One commit, something should be in the authors list @@ -77,10 +97,8 @@ class TestPackagingInGitRepoWithoutCommit(base.BaseTestCase): def setUp(self): super(TestPackagingInGitRepoWithoutCommit, self).setUp() - self._run_cmd('git', ['init', '.']) - self._run_cmd('git', ['add', '.']) - self.run_setup('sdist') - return + self.useFixture(TestRepo(self.package_dir)) + self.run_setup('sdist', allow_fail=False) def test_authors(self): # No commits, no authors in list @@ -99,8 +117,7 @@ class TestPackagingInPlainDirectory(base.BaseTestCase): def setUp(self): super(TestPackagingInPlainDirectory, self).setUp() - self.run_setup('sdist') - return + self.run_setup('sdist', allow_fail=False) def test_authors(self): # Not a git repo, no AUTHORS file created @@ -126,3 +143,17 @@ class TestPresenceOfGit(base.BaseTestCase): '_run_shell_command') as _command: _command.side_effect = OSError self.assertEqual(False, packaging._git_is_installed()) + + +class TestNestedRequirements(base.BaseTestCase): + + def test_nested_requirement(self): + tempdir = tempfile.mkdtemp() + requirements = os.path.join(tempdir, 'requirements.txt') + nested = os.path.join(tempdir, 'nested.txt') + with open(requirements, 'w') as f: + f.write('-r ' + nested) + with open(nested, 'w') as f: + f.write('pbr') + result = packaging.parse_requirements([requirements]) + self.assertEqual(result, ['pbr']) diff --git a/awx/lib/site-packages/pbr/tests/test_setup.py b/awx/lib/site-packages/pbr/tests/test_setup.py index 960de997d8..44436df5e3 100644 --- a/awx/lib/site-packages/pbr/tests/test_setup.py +++ b/awx/lib/site-packages/pbr/tests/test_setup.py @@ -151,7 +151,7 @@ class GitLogsTest(base.BaseTestCase): co_author_by = u"Co-authored-by: " + co_author git_log_cmd = ( - "git --git-dir=%s log --use-mailmap --format=%%aN <%%aE>" + "git --git-dir=%s log --format=%%aN <%%aE>" % self.git_dir) git_co_log_cmd = ("git --git-dir=%s log" % self.git_dir) git_top_level = "git rev-parse --show-toplevel" @@ -304,26 +304,6 @@ class ParseRequirementsTest(base.BaseTestCase): fh.write("-f foobar") self.assertEqual([], packaging.parse_requirements([self.tmp_file])) - def test_parse_requirements_removes_argparse(self): - with open(self.tmp_file, 'w') as fh: - fh.write("argparse") - if sys.version_info >= (2, 7): - self.assertEqual([], packaging.parse_requirements([self.tmp_file])) - - def test_parse_requirements_removes_versioned_ordereddict(self): - self.useFixture(fixtures.MonkeyPatch('sys.version_info', (2, 7))) - with open(self.tmp_file, 'w') as fh: - fh.write("ordereddict==1.0.1") - self.assertEqual([], packaging.parse_requirements([self.tmp_file])) - - def test_parse_requirements_keeps_versioned_ordereddict(self): - self.useFixture(fixtures.MonkeyPatch('sys.version_info', (2, 6))) - with open(self.tmp_file, 'w') as fh: - fh.write("ordereddict==1.0.1") - self.assertEqual([ - "ordereddict==1.0.1"], - packaging.parse_requirements([self.tmp_file])) - def test_parse_requirements_override_with_env(self): with open(self.tmp_file, 'w') as fh: fh.write("foo\nbar")