Merge pull request #4516 from ryanpetrello/py2

support the new CLI in py2 *and* py3

Reviewed-by: https://github.com/apps/softwarefactory-project-zuul

Makefile

@@ -376,7 +376,7 @@ test:
 		. $(VENV_BASE)/awx/bin/activate; \
 	fi; \
 	PYTHONDONTWRITEBYTECODE=1 py.test -p no:cacheprovider -n auto $(TEST_DIRS)
-	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py3
+	cd awxkit && $(VENV_BASE)/awx/bin/tox -re py2,py3
 	awx-manage check_migrations --dry-run --check  -n 'vNNN_missing_migration_file'
 
 test_unit:

awxkit/awxkit/__init__.py

@@ -1,4 +1,4 @@
-from .api import pages, client, resources # NOQA
+from awxkit.api import pages, client, resources # NOQA
-from .config import config # NOQA
+from awxkit.config import config # NOQA
-from . import awx # NOQA
+from awxkit import awx # NOQA
-from .ws import WSClient # NOQA
+from awxkit.ws import WSClient # NOQA

awxkit/awxkit/api/mixins/has_status.py

@@ -1,6 +1,8 @@
 from datetime import datetime
 import json
 
+import six
+
 from awxkit.utils import poll_until
 
 
@@ -43,7 +45,7 @@ class HasStatus(object):
         return self.wait_until_status(self.started_statuses, interval=interval, timeout=timeout)
 
     def assert_status(self, status_list, msg=None):
-        if isinstance(status_list, str):
+        if isinstance(status_list, six.text_type):
             status_list = [status_list]
         if self.status in status_list:
             # include corner cases in is_successful logic

awxkit/awxkit/api/pages/page.py

@@ -1,10 +1,11 @@
-import http.client
 import inspect
 import logging
 import json
 import re
 
 from requests import Response
+import six
+from six.moves import http_client as http
 
 from awxkit.utils import (
     PseudoNamespace,
@@ -12,7 +13,8 @@ from awxkit.utils import (
     are_same_endpoint,
     super_dir_set,
     suppress,
-    is_list_or_tuple
+    is_list_or_tuple,
+    to_str
 )
 from awxkit.api.client import Connection
 from awxkit.api.registry import URLRegistry
@@ -167,7 +169,11 @@ class Page(object):
     @classmethod
     def from_json(cls, raw):
         resp = Response()
-        resp._content = bytes(json.dumps(raw), 'utf-8')
+        data = json.dumps(raw)
+        if six.PY3:
+            resp._content = bytes(data, 'utf-8')
+        else:
+            resp._content = data
         resp.encoding = 'utf-8'
         resp.status_code = 200
         return cls(r=resp)
@@ -199,16 +205,16 @@ class Page(object):
                     "Unable to parse JSON response ({0.status_code}): {1} - '{2}'".format(response, e, text))
 
         exc_str = "%s (%s) received" % (
-            http.client.responses[response.status_code], response.status_code)
+            http.responses[response.status_code], response.status_code)
 
         exception = exception_from_status_code(response.status_code)
         if exception:
             raise exception(exc_str, data)
 
         if response.status_code in (
-                http.client.OK,
+                http.OK,
-                http.client.CREATED,
+                http.CREATED,
-                http.client.ACCEPTED):
+                http.ACCEPTED):
 
             # Not all JSON responses include a URL.  Grab it from the request
             # object, if needed.
@@ -235,7 +241,7 @@ class Page(object):
                 r=response,
                 ds=ds)
 
-        elif response.status_code == http.client.FORBIDDEN:
+        elif response.status_code == http.FORBIDDEN:
             if is_license_invalid(response):
                 raise exc.LicenseInvalid(exc_str, data)
             elif is_license_exceeded(response):
@@ -243,7 +249,7 @@ class Page(object):
             else:
                 raise exc.Forbidden(exc_str, data)
 
-        elif response.status_code == http.client.BAD_REQUEST:
+        elif response.status_code == http.BAD_REQUEST:
             if is_license_invalid(response):
                 raise exc.LicenseInvalid(exc_str, data)
             if is_duplicate_error(response):
@@ -314,14 +320,14 @@ class Page(object):
         return page_cls(self.connection, endpoint=endpoint).get(**kw)
 
 
-_exception_map = {http.client.NO_CONTENT: exc.NoContent,
+_exception_map = {http.NO_CONTENT: exc.NoContent,
-                  http.client.NOT_FOUND: exc.NotFound,
+                  http.NOT_FOUND: exc.NotFound,
-                  http.client.INTERNAL_SERVER_ERROR: exc.InternalServerError,
+                  http.INTERNAL_SERVER_ERROR: exc.InternalServerError,
-                  http.client.BAD_GATEWAY: exc.BadGateway,
+                  http.BAD_GATEWAY: exc.BadGateway,
-                  http.client.METHOD_NOT_ALLOWED: exc.MethodNotAllowed,
+                  http.METHOD_NOT_ALLOWED: exc.MethodNotAllowed,
-                  http.client.UNAUTHORIZED: exc.Unauthorized,
+                  http.UNAUTHORIZED: exc.Unauthorized,
-                  http.client.PAYMENT_REQUIRED: exc.PaymentRequired,
+                  http.PAYMENT_REQUIRED: exc.PaymentRequired,
-                  http.client.CONFLICT: exc.Conflict}
+                  http.CONFLICT: exc.Conflict}
 
 
 def exception_from_status_code(status_code):
@@ -376,10 +382,10 @@ class PageList(object):
 class TentativePage(str):
 
     def __new__(cls, endpoint, connection):
-        return super(TentativePage, cls).__new__(cls, endpoint)
+        return super(TentativePage, cls).__new__(cls, to_str(endpoint))
 
     def __init__(self, endpoint, connection):
-        self.endpoint = endpoint
+        self.endpoint = to_str(endpoint)
         self.connection = connection
 
     def _create(self):

awxkit/awxkit/cli/__init__.py

@@ -6,6 +6,7 @@ import yaml
 from requests.exceptions import ConnectionError, SSLError
 
 from .client import CLI
+from awxkit.utils import to_str
 from awxkit.exceptions import Unauthorized, Common
 from awxkit.cli.utils import cprint
 
@@ -46,7 +47,14 @@ def run(stdout=sys.stdout, stderr=sys.stderr, argv=[]):
         if cli.get_config('format') == 'json':
             json.dump(e.msg, sys.stdout)
         elif cli.get_config('format') == 'yaml':
-            sys.stdout.write(yaml.dump(e.msg))
+            sys.stdout.write(to_str(
+                yaml.safe_dump(
+                    e.msg,
+                    default_flow_style=False,
+                    encoding='utf-8',
+                    allow_unicode=True
+                )
+            ))
         sys.exit(1)
     except Exception as e:
         if cli.verbose:

awxkit/awxkit/cli/client.py

@@ -1,9 +1,12 @@
+from __future__ import print_function
+
 import logging
 import os
 import pkg_resources
 import sys
 
 from requests.exceptions import RequestException
+import six
 
 from .custom import handle_custom_actions
 from .format import (add_authentication_arguments,
@@ -160,10 +163,18 @@ class CLI(object):
                 changed=self.original_action in ('modify', 'create')
             )
             if formatted:
-                print(formatted, file=self.stdout)
+                print(utils.to_str(formatted), file=self.stdout)
         else:
             self.parser.print_help()
 
+            if six.PY2 and not self.help:
+                # Unfortunately, argparse behavior between py2 and py3
+                # changed in a notable way when required subparsers
+                # have invalid (or missing) arguments specified
+                # see: https://github.com/python/cpython/commit/f97c59aaba2d93e48cbc6d25f7ff9f9c87f8d0b2
+                print('\nargument resource: invalid choice')
+                raise SystemExit(2)
+
     def parse_action(self, page, from_sphinx=False):
         """Perform an HTTP OPTIONS request
 

awxkit/awxkit/cli/custom.py

@@ -1,3 +1,5 @@
+from six import with_metaclass
+
 from .stdout import monitor, monitor_workflow
 from .utils import CustomRegistryMeta, color_enabled
 
@@ -17,7 +19,7 @@ class CustomActionRegistryMeta(CustomRegistryMeta):
         return ' '.join([self.resource, self.action])
 
 
-class CustomAction(object, metaclass=CustomActionRegistryMeta):
+class CustomAction(with_metaclass(CustomActionRegistryMeta)):
     """Base class for defining a custom action for a resource."""
 
     def __init__(self, page):

awxkit/awxkit/cli/format.py

@@ -1,6 +1,7 @@
 import json
 from distutils.util import strtobool
 
+import six
 import yaml
 
 from awxkit.cli.utils import colored
@@ -79,7 +80,7 @@ def add_output_formatting_arguments(parser, env):
 def format_response(response, fmt='json', filter='.', changed=False):
     if response is None:
         return  # HTTP 204
-    if isinstance(response, str):
+    if isinstance(response, six.text_type):
         return response
 
     if 'results' in response.__dict__:
@@ -113,7 +114,7 @@ def format_jq(output, fmt):
     results = []
     for x in jq.jq(fmt).transform(output, multiple_output=True):
         if x not in (None, ''):
-            if isinstance(x, str):
+            if isinstance(x, six.text_type):
                 results.append(x)
             else:
                 results.append(json.dumps(x))
@@ -126,9 +127,11 @@ def format_json(output, fmt):
 
 def format_yaml(output, fmt):
     output = json.loads(json.dumps(output))
-    return yaml.dump(
+    return yaml.safe_dump(
         output,
-        default_flow_style=False
+        default_flow_style=False,
+        encoding='utf-8',
+        allow_unicode=True
     )
 
 

awxkit/awxkit/cli/resource.py

@@ -1,5 +1,7 @@
 import os
 
+from six import PY3, with_metaclass
+
 from awxkit import api, config
 from awxkit.api.pages import Page
 from awxkit.cli.format import format_response, add_authentication_arguments
@@ -40,7 +42,7 @@ DEPRECATED_RESOURCES_REVERSE = dict(
 )
 
 
-class CustomCommand(object, metaclass=CustomRegistryMeta):
+class CustomCommand(with_metaclass(CustomRegistryMeta)):
     """Base class for implementing custom commands.
 
     Custom commands represent static code which should run - they are
@@ -121,15 +123,28 @@ def parse_resource(client, skip_deprecated=False):
             if k in ('dashboard',):
                 # the Dashboard API is deprecated and not supported
                 continue
-            aliases = []
+
-            if not skip_deprecated:
+            # argparse aliases are *only* supported in Python3 (not 2.7)
+            kwargs = {}
+            if not skip_deprecated and PY3:
                 if k in DEPRECATED_RESOURCES:
-                    aliases = [DEPRECATED_RESOURCES[k]]
+                    kwargs['aliases'] = [DEPRECATED_RESOURCES[k]]
             client.subparsers[k] = subparsers.add_parser(
-                k, help='', aliases=aliases
+                k, help='', **kwargs
             )
 
-    resource = client.parser.parse_known_args()[0].resource
+    try:
+        resource = client.parser.parse_known_args()[0].resource
+    except SystemExit:
+        if PY3:
+            raise
+        else:
+            # Unfortunately, argparse behavior between py2 and py3
+            # changed in a notable way when required subparsers
+            # have invalid (or missing) arguments specified
+            # see: https://github.com/python/cpython/commit/f97c59aaba2d93e48cbc6d25f7ff9f9c87f8d0b2
+            # In py2, this raises a SystemExit; which we want to _ignore_
+            resource = None
     if resource in DEPRECATED_RESOURCES.values():
         client.argv[
             client.argv.index(resource)

awxkit/awxkit/cli/stdout.py

@@ -1,8 +1,12 @@
+# -*- coding: utf-8 -*-
+from __future__ import print_function
+
 import sys
 
 import time
 
 from .utils import cprint, color_enabled, STATUS_COLORS
+from awxkit.utils import to_str
 
 
 def monitor_workflow(response, session, print_stdout=True, timeout=None,
@@ -25,6 +29,7 @@ def monitor_workflow(response, session, print_stdout=True, timeout=None,
                 sys.stdout.write('\x1b[2K')
 
         for result in results:
+            result['name'] = to_str(result['name'])
             if print_stdout:
                 print(' ↳ {id} - {name} '.format(**result), end='')
                 status = result['status']
@@ -39,7 +44,7 @@ def monitor_workflow(response, session, print_stdout=True, timeout=None,
         cprint('------Starting Standard Out Stream------', 'red')
 
     if print_stdout:
-        print('Launching {}...'.format(get().json.name))
+        print('Launching {}...'.format(to_str(get().json.name)))
 
     started = time.time()
     seen = set()
@@ -84,7 +89,7 @@ def monitor(response, session, print_stdout=True, timeout=None, interval=.25):
                 # skip it for now and wait until the prior lines arrive and are
                 # printed
                 continue
-            stdout = result.get('stdout')
+            stdout = to_str(result.get('stdout'))
             if stdout and print_stdout:
                 print(stdout)
             next_line = result['end_line']

awxkit/awxkit/cli/utils.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 from argparse import ArgumentParser
 import os
 import sys

awxkit/awxkit/utils/__init__.py

@@ -10,6 +10,7 @@ import sys
 import re
 import os
 
+import six
 import yaml
 
 from awxkit.words import words
@@ -132,7 +133,7 @@ class PseudoNamespace(dict):
 
 
 def is_relative_endpoint(candidate):
-    return isinstance(candidate, (str,)) and candidate.startswith('/api/')
+    return isinstance(candidate, (six.text_type,)) and candidate.startswith('/api/')
 
 
 def is_class_or_instance(obj, cls):
@@ -320,6 +321,22 @@ def update_payload(payload, fields, kwargs):
     return payload
 
 
+def to_str(obj):
+    if six.PY3:
+        if isinstance(obj, bytes):
+            return obj.decode('utf-8')
+        return obj
+    if not isinstance(obj, six.text_type):
+        try:
+            return str(obj)
+        except UnicodeDecodeError:
+            try:
+                obj = six.text_type(obj, 'utf8')
+            except UnicodeDecodeError:
+                obj = obj.decode('latin1')
+    return obj.encode('utf8')
+
+
 def to_bool(obj):
     if isinstance(obj, (str,)):
         return obj.lower() not in ('false', 'off', 'no', 'n', '0', '')

awxkit/awxkit/ws.py

@@ -1,11 +1,12 @@
-from queue import Queue, Empty
 import time
 import threading
 import logging
 import atexit
 import json
 import ssl
-import urllib.parse
+
+from six.moves.queue import Queue, Empty
+from six.moves.urllib.parse import urlparse
 
 from awxkit.config import config
 
@@ -55,7 +56,7 @@ class WSClient(object):
         import websocket
 
         if not hostname:
-            result = urllib.parse.urlparse(config.base_url)
+            result = urlparse(config.base_url)
             secure = result.scheme == 'https'
             port = result.port
             if port is None:

awxkit/requirements.txt

@@ -1,2 +1,3 @@
 PyYAML
 requests
+six

awxkit/setup.py

@@ -67,7 +67,7 @@ setup(
     },
     include_package_data=True,
     install_requires=requirements,
-    python_requires=">= 3.5",
+    python_requires=">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*",
     extras_require={
         'formatting': ['jq', 'tabulate'],
         'websockets': ['websocket-client>0.54.0'],

awxkit/test/cli/test_client.py

@@ -50,8 +50,14 @@ def test_list_resources(capfd, resource):
     cli.parse_args(['awx {}'.format(resource)])
     cli.connect()
 
-    cli.parse_resource()
+    try:
-    out, err = capfd.readouterr()
+        cli.parse_resource()
+        out, err = capfd.readouterr()
+    except SystemExit:
+        # python2 argparse raises SystemExit for invalid/missing required args,
+        # py3 doesn't
+        _, out = capfd.readouterr()
+
     assert "usage:" in out
     for snippet in (
         '--conf.host https://example.awx.org]',

awxkit/test/cli/test_options.py

@@ -1,7 +1,10 @@
 import argparse
 import json
 import unittest
-from io import StringIO
+try:
+    from StringIO import StringIO
+except ImportError:
+    from io import StringIO
 
 import pytest
 from requests import Response

awxkit/test/test_credentials.py

@@ -1,4 +1,7 @@
-from unittest.mock import patch
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 import pytest
 
 

awxkit/test/test_utils.py

@@ -1,8 +1,13 @@
 # -*- coding: utf-8 -*-
 from datetime import datetime
+import sys
 
-from unittest import mock
+try:
+    from unittest import mock
+except ImportError:
+    import mock
 import pytest
+import six
 
 from awxkit import utils
 from awxkit import exceptions as exc
@@ -73,11 +78,19 @@ def test_load_invalid_json_or_yaml(inp):
 
 
 @pytest.mark.parametrize('non_ascii', [True, False])
+@pytest.mark.skipif(
+    sys.version_info < (3, 6),
+    reason='this is only intended to be used in py3, not the CLI'
+)
 def test_random_titles_are_unicode(non_ascii):
-    assert isinstance(utils.random_title(non_ascii=non_ascii), str)
+    assert isinstance(utils.random_title(non_ascii=non_ascii), six.text_type)
 
 
 @pytest.mark.parametrize('non_ascii', [True, False])
+@pytest.mark.skipif(
+    sys.version_info < (3, 6),
+    reason='this is only intended to be used in py3, not the CLI'
+)
 def test_random_titles_generates_correct_characters(non_ascii):
     title = utils.random_title(non_ascii=non_ascii)
     if non_ascii:

awxkit/test/test_ws.py

@@ -1,7 +1,10 @@
 # -*- coding: utf-8 -*-
 from collections import namedtuple
 
-from unittest.mock import patch
+try:
+    from unittest.mock import patch
+except ImportError:
+    from mock import patch
 import pytest
 
 from awxkit.ws import WSClient

awxkit/tox.ini

@@ -14,6 +14,7 @@ setenv =
 deps =
     websocket-client
     coverage
+    mock
     pytest
     pytest-mock