Upgrade pyrax to 1.9.0

awx/lib/site-packages/README

@@ -43,7 +43,7 @@ pexpect==3.3 (pexpect/*, excluded pxssh.py, fdpexpect.py, FSM.py, screen.py,
               ANSI.py)
 pip==1.5.4 (pip/*, excluded bin/pip*)
 prettytable==0.7.2 (prettytable.py)
-pyrax==1.7.2 (pyrax/*)
+pyrax==1.9.0 (pyrax/*)
 python-dateutil==2.2 (dateutil/*)
 python-novaclient==2.17.0 (novaclient/*, excluded bin/nova)
 python-swiftclient==2.0.3 (swiftclient/*, excluded bin/swift)

awx/lib/site-packages/pyrax/__init__.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -27,22 +27,17 @@ built on the Rackspace / OpenStack Cloud.<br />
 The source code for <b>pyrax</b> can be found at:
 
 http://github.com/rackspace/pyrax
-
-\package cf_wrapper
-
-This module wraps <b>swiftclient</b>, the Python client for OpenStack / Swift,
-providing an object-oriented interface to the Swift object store.
-
-It also adds in CDN functionality that is Rackspace-specific.
 """
+
+from __future__ import absolute_import
 from functools import wraps
 import inspect
 import logging
 import os
+import re
+import six.moves.configparser as ConfigParser
 import warnings
 
-from six.moves import configparser
-
 # keyring is an optional import
 try:
     import keyring
@@ -59,21 +54,22 @@ try:
     from . import http
     from . import version
 
-    import cf_wrapper.client as _cf
     from novaclient import exceptions as _cs_exceptions
     from novaclient import auth_plugin as _cs_auth_plugin
+    from novaclient.shell import OpenStackComputeShell as _cs_shell
     from novaclient.v1_1 import client as _cs_client
     from novaclient.v1_1.servers import Server as CloudServer
 
-    from autoscale import AutoScaleClient
+    from .autoscale import AutoScaleClient
-    from clouddatabases import CloudDatabaseClient
+    from .clouddatabases import CloudDatabaseClient
-    from cloudloadbalancers import CloudLoadBalancerClient
+    from .cloudloadbalancers import CloudLoadBalancerClient
-    from cloudblockstorage import CloudBlockStorageClient
+    from .cloudblockstorage import CloudBlockStorageClient
-    from clouddns import CloudDNSClient
+    from .clouddns import CloudDNSClient
-    from cloudnetworks import CloudNetworkClient
+    from .cloudnetworks import CloudNetworkClient
-    from cloudmonitoring import CloudMonitorClient
+    from .cloudmonitoring import CloudMonitorClient
-    from image import ImageClient
+    from .image import ImageClient
-    from queueing import QueueClient
+    from .object_storage import StorageClient
+    from .queueing import QueueClient
 except ImportError:
     # See if this is the result of the importing of version.py in setup.py
     callstack = inspect.stack()
@@ -118,6 +114,8 @@ regions = tuple()
 services = tuple()
 
 _client_classes = {
+        "compute": _cs_client.Client,
+        "object_store": StorageClient,
         "database": CloudDatabaseClient,
         "load_balancer": CloudLoadBalancerClient,
         "volume": CloudBlockStorageClient,
@@ -168,7 +166,7 @@ class Settings(object):
             "verify_ssl": "CLOUD_VERIFY_SSL",
             "use_servicenet": "USE_SERVICENET",
             }
-    _settings = {"default": dict.fromkeys(env_dct.keys())}
+    _settings = {"default": dict.fromkeys(list(env_dct.keys()))}
     _default_set = False
 
 
@@ -181,8 +179,10 @@ class Settings(object):
         if env is None:
             env = self.environment
         try:
-            return self._settings[env][key]
+            ret = self._settings[env][key]
         except KeyError:
+            ret = None
+        if ret is None:
             # See if it's set in the environment
             if key == "identity_class":
                 # This is defined via the identity_type
@@ -193,9 +193,10 @@ class Settings(object):
             else:
                 env_var = self.env_dct.get(key)
             try:
-                return os.environ[env_var]
+                ret = os.environ[env_var]
             except KeyError:
-                return None
+                ret = None
+        return ret
 
 
     def set(self, key, val, env=None):
@@ -210,7 +211,7 @@ class Settings(object):
         else:
             if env not in self._settings:
                 raise exc.EnvironmentNotFound("There is no environment named "
-                "'%s'." % env)
+                        "'%s'." % env)
         dct = self._settings[env]
         if key not in dct:
             raise exc.InvalidSetting("The setting '%s' is not defined." % key)
@@ -257,7 +258,7 @@ class Settings(object):
 
     @property
     def environments(self):
-        return self._settings.keys()
+        return list(self._settings.keys())
 
 
     def read_config(self, config_file):
@@ -265,17 +266,17 @@ class Settings(object):
         Parses the specified configuration file and stores the values. Raises
         an InvalidConfigurationFile exception if the file is not well-formed.
         """
-        cfg = configparser.SafeConfigParser()
+        cfg = ConfigParser.SafeConfigParser()
         try:
             cfg.read(config_file)
-        except configparser.MissingSectionHeaderError as e:
+        except ConfigParser.MissingSectionHeaderError as e:
             # The file exists, but doesn't have the correct format.
             raise exc.InvalidConfigurationFile(e)
 
         def safe_get(section, option, default=None):
             try:
                 return cfg.get(section, option)
-            except (configparser.NoSectionError, configparser.NoOptionError):
+            except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
                 return default
 
         # A common mistake is including credentials in the config file. If any
@@ -297,8 +298,9 @@ class Settings(object):
             dct = self._settings[section_name] = {}
             dct["region"] = safe_get(section, "region", default_region)
             ityp = safe_get(section, "identity_type")
-            dct["identity_type"] = _id_type(ityp)
+            if ityp:
-            dct["identity_class"] = _import_identity(ityp)
+                dct["identity_type"] = _id_type(ityp)
+                dct["identity_class"] = _import_identity(ityp)
             # Handle both the old and new names for this setting.
             debug = safe_get(section, "debug")
             if debug is None:
@@ -377,18 +379,46 @@ def set_default_region(region):
     default_region = region
 
 
-def _create_identity():
+def create_context(id_type=None, env=None, username=None, password=None,
+        tenant_id=None, tenant_name=None, api_key=None, verify_ssl=None):
+    """
+    Returns an instance of the specified identity class, or if none is
+    specified, an instance of the current setting for 'identity_class'.
+
+    You may optionally set the environment by passing the name of that
+    environment in the 'env' parameter.
+    """
+    if env:
+        set_environment(env)
+    return _create_identity(id_type=id_type, username=username,
+            password=password, tenant_id=tenant_id, tenant_name=tenant_name,
+            api_key=api_key, verify_ssl=verify_ssl, return_context=True)
+
+
+def _create_identity(id_type=None, username=None, password=None, tenant_id=None,
+            tenant_name=None, api_key=None, verify_ssl=None,
+            return_context=False):
     """
     Creates an instance of the current identity_class and assigns it to the
-    module-level name 'identity'.
+    module-level name 'identity' by default. If 'return_context' is True, the
+    module-level 'identity' is untouched, and instead the instance is returned.
     """
-    global identity
+    if id_type:
-    cls = settings.get("identity_class")
+        cls = _import_identity(id_type)
+    else:
+        cls = settings.get("identity_class")
     if not cls:
         raise exc.IdentityClassNotDefined("No identity class has "
                 "been defined for the current environment.")
-    verify_ssl = get_setting("verify_ssl")
+    if verify_ssl is None:
-    identity = cls(verify_ssl=verify_ssl)
+        verify_ssl = get_setting("verify_ssl")
+    context = cls(username=username, password=password, tenant_id=tenant_id,
+            tenant_name=tenant_name, api_key=api_key, verify_ssl=verify_ssl)
+    if return_context:
+        return context
+    else:
+        global identity
+        identity = context
 
 
 def _assure_identity(fnc):
@@ -412,13 +442,16 @@ def _require_auth(fnc):
     return _wrapped
 
 
-@_assure_identity
+def _safe_region(region=None, context=None):
-def _safe_region(region=None):
     """Value to use when no region is specified."""
     ret = region or settings.get("region")
+    context = context or identity
     if not ret:
         # Nothing specified; get the default from the identity object.
-        ret = identity.get_default_region()
+        if not context:
+            _create_identity()
+            context = identity
+        ret = context.get_default_region()
     if not ret:
         # Use the first available region
         try:
@@ -434,8 +467,11 @@ def auth_with_token(token, tenant_id=None, tenant_name=None, region=None):
     If you already have a valid token and either a tenant ID or name, you can
     call this to configure the identity and available services.
     """
+    global regions, services
     identity.auth_with_token(token, tenant_id=tenant_id,
             tenant_name=tenant_name)
+    regions = tuple(identity.regions)
+    services = tuple(identity.services.keys())
     connect_to_services(region=region)
 
 
@@ -448,13 +484,15 @@ def set_credentials(username, api_key=None, password=None, region=None,
     If the region is passed, it will authenticate against the proper endpoint
     for that region, and set the default region for connections.
     """
+    global regions, services
     pw_key = password or api_key
     region = _safe_region(region)
     tenant_id = tenant_id or settings.get("tenant_id")
     identity.set_credentials(username=username, password=pw_key,
-            tenant_id=tenant_id, region=region)
+            tenant_id=tenant_id, region=region, authenticate=authenticate)
-    if authenticate:
+    regions = tuple(identity.regions)
-        _auth_and_connect(region=region)
+    services = tuple(identity.services.keys())
+    connect_to_services(region=region)
 
 
 @_assure_identity
@@ -478,10 +516,13 @@ def set_credential_file(cred_file, region=None, authenticate=True):
     If the region is passed, it will authenticate against the proper endpoint
     for that region, and set the default region for connections.
     """
+    global regions, services
     region = _safe_region(region)
-    identity.set_credential_file(cred_file, region=region)
+    identity.set_credential_file(cred_file, region=region,
-    if authenticate:
+            authenticate=authenticate)
-        _auth_and_connect(region=region)
+    regions = tuple(identity.regions)
+    services = tuple(identity.services.keys())
+    connect_to_services(region=region)
 
 
 def keyring_auth(username=None, region=None, authenticate=True):
@@ -514,23 +555,6 @@ def keyring_auth(username=None, region=None, authenticate=True):
             authenticate=authenticate)
 
 
-def _auth_and_connect(region=None, connect=True):
-    """
-    Handles the call to authenticate, and if successful, connects to the
-    various services.
-    """
-    global default_region
-    identity.authenticated = False
-    default_region = region or default_region
-    try:
-        identity.authenticate()
-    except exc.AuthenticationFailed:
-        clear_credentials()
-        raise
-    if connect:
-        connect_to_services(region=region)
-
-
 @_assure_identity
 def authenticate(connect=True):
     """
@@ -545,19 +569,11 @@ def authenticate(connect=True):
     Normally after successful authentication, connections to the various
     services will be made. However, passing False to the `connect` parameter
     will skip the service connection step.
-    """
-    _auth_and_connect(connect=connect)
 
-
+    The 'connect' parameter is retained for backwards compatibility. It no
-def plug_hole_in_swiftclient_auth(clt, url):
+    longer has any effect.
     """
-    This is necessary because swiftclient has an issue when a token expires and
+    identity.authenticate()
-    it needs to re-authenticate against Rackspace auth. It is a temporary
-    workaround until we can fix swiftclient.
-    """
-    conn = clt.connection
-    conn.token = identity.token
-    conn.url = url
 
 
 def clear_credentials():
@@ -610,46 +626,52 @@ def connect_to_services(region=None):
     queues = connect_to_queues(region=region)
 
 
-def _get_service_endpoint(svc, region=None, public=True):
+def _get_service_endpoint(context, svc, region=None, public=True):
     """
     Parses the services dict to get the proper endpoint for the given service.
     """
     region = _safe_region(region)
-    url_type = {True: "public_url", False: "internal_url"}[public]
+    # If a specific context is passed, use that. Otherwise, use the global
-    ep = identity.services.get(svc, {}).get("endpoints", {}).get(
+    # identity reference.
-            region, {}).get(url_type)
+    context = context or identity
+    url_type = {True: "public", False: "private"}[public]
+    svc_obj = context.services.get(svc)
+    if not svc_obj:
+        return None
+    ep = svc_obj.endpoints.get(region, {}).get(url_type)
     if not ep:
         # Try the "ALL" region, and substitute the actual region
-        ep = identity.services.get(svc, {}).get("endpoints", {}).get(
+        ep = svc_obj.endpoints.get("ALL", {}).get(url_type)
-                "ALL", {}).get(url_type)
     return ep
 
 
-@_require_auth
+def connect_to_cloudservers(region=None, context=None, **kwargs):
-def connect_to_cloudservers(region=None, **kwargs):
     """Creates a client for working with cloud servers."""
+    context = context or identity
     _cs_auth_plugin.discover_auth_systems()
     id_type = get_setting("identity_type")
     if id_type != "keystone":
         auth_plugin = _cs_auth_plugin.load_plugin(id_type)
     else:
         auth_plugin = None
-    region = _safe_region(region)
+    region = _safe_region(region, context=context)
-    mgt_url = _get_service_endpoint("compute", region)
+    mgt_url = _get_service_endpoint(context, "compute", region)
     cloudservers = None
     if not mgt_url:
         # Service is not available
         return
     insecure = not get_setting("verify_ssl")
-    cloudservers = _cs_client.Client(identity.username, identity.password,
+    cs_shell = _cs_shell()
-            project_id=identity.tenant_id, auth_url=identity.auth_endpoint,
+    extensions = cs_shell._discover_extensions("1.1")
+    cloudservers = _cs_client.Client(context.username, context.password,
+            project_id=context.tenant_id, auth_url=context.auth_endpoint,
             auth_system=id_type, region_name=region, service_type="compute",
-            auth_plugin=auth_plugin, insecure=insecure,
+            auth_plugin=auth_plugin, insecure=insecure, extensions=extensions,
             http_log_debug=_http_debug, **kwargs)
     agt = cloudservers.client.USER_AGENT
     cloudservers.client.USER_AGENT = _make_agent_name(agt)
     cloudservers.client.management_url = mgt_url
-    cloudservers.client.auth_token = identity.token
+    cloudservers.client.auth_token = context.token
     cloudservers.exceptions = _cs_exceptions
     # Add some convenience methods
     cloudservers.list_images = cloudservers.images.list
@@ -672,54 +694,50 @@ def connect_to_cloudservers(region=None, **kwargs):
         return [image for image in cloudservers.images.list()
                 if hasattr(image, "server")]
 
+    def find_images_by_name(expr):
+        """
+        Returns a list of images whose name contains the specified expression.
+        The value passed is treated as a regular expression, allowing for more
+        specific searches than simple wildcards. The matching is done in a
+        case-insensitive manner.
+        """
+        return [image for image in cloudservers.images.list()
+                if re.search(expr, image.name, re.I)]
+
     cloudservers.list_base_images = list_base_images
     cloudservers.list_snapshots = list_snapshots
+    cloudservers.find_images_by_name = find_images_by_name
+    cloudservers.identity = identity
     return cloudservers
 
 
-@_require_auth
 def connect_to_cloudfiles(region=None, public=None):
-    """
+    """Creates a client for working with CloudFiles/Swift."""
-    Creates a client for working with cloud files. The default is to connect
-    to the public URL; if you need to work with the ServiceNet connection, pass
-    False to the 'public' parameter or set the "use_servicenet" setting to True.
-    """
     if public is None:
         is_public = not bool(get_setting("use_servicenet"))
     else:
         is_public = public
-
+    ret = _create_client(ep_name="object_store", region=region,
-    region = _safe_region(region)
+            public=is_public)
-    cf_url = _get_service_endpoint("object_store", region, public=is_public)
+    if ret:
-    cloudfiles = None
+        # Add CDN endpoints, if available
-    if not cf_url:
+        region = _safe_region(region)
-        # Service is not available
+        ret.cdn_management_url = _get_service_endpoint(None, "object_cdn",
-        return
+                region, public=is_public)
-    cdn_url = _get_service_endpoint("object_cdn", region)
+    return ret
-    ep_type = {True: "publicURL", False: "internalURL"}[is_public]
-    opts = {"tenant_id": identity.tenant_name, "auth_token": identity.token,
-            "endpoint_type": ep_type, "tenant_name": identity.tenant_name,
-            "object_storage_url": cf_url, "object_cdn_url": cdn_url,
-            "region_name": region}
-    verify_ssl = get_setting("verify_ssl")
-    cloudfiles = _cf.CFClient(identity.auth_endpoint, identity.username,
-            identity.password, tenant_name=identity.tenant_name,
-            preauthurl=cf_url, preauthtoken=identity.token, auth_version="2",
-            os_options=opts, verify_ssl=verify_ssl, http_log_debug=_http_debug)
-    cloudfiles.user_agent = _make_agent_name(cloudfiles.user_agent)
-    return cloudfiles
 
 
 @_require_auth
 def _create_client(ep_name, region, public=True):
     region = _safe_region(region)
-    ep = _get_service_endpoint(ep_name.split(":")[0], region, public=public)
+    ep = _get_service_endpoint(None, ep_name.split(":")[0], region,
+            public=public)
     if not ep:
         return
     verify_ssl = get_setting("verify_ssl")
     cls = _client_classes[ep_name]
-    client = cls(region_name=region, management_url=ep, verify_ssl=verify_ssl,
+    client = cls(identity, region_name=region, management_url=ep,
-            http_log_debug=_http_debug)
+            verify_ssl=verify_ssl, http_log_debug=_http_debug)
     client.user_agent = _make_agent_name(client.user_agent)
     return client
 
@@ -769,34 +787,29 @@ def connect_to_queues(region=None, public=True):
     return _create_client(ep_name="queues", region=region, public=public)
 
 
+def client_class_for_service(service):
+    """
+    Returns the client class registered for the given service, or None if there
+    is no such service, or if no class has been registered.
+    """
+    return _client_classes.get(service)
+
+
 def get_http_debug():
     return _http_debug
 
 
-@_assure_identity
 def set_http_debug(val):
     global _http_debug
     _http_debug = val
     # Set debug on the various services
-    identity.http_log_debug = val
+    if identity:
+        identity.http_log_debug = val
     for svc in (cloudservers, cloudfiles, cloud_loadbalancers,
             cloud_blockstorage, cloud_databases, cloud_dns, cloud_networks,
             autoscale, images, queues):
         if svc is not None:
             svc.http_log_debug = val
-    # Need to manually add/remove the debug handler for swiftclient
-    swift_logger = _cf._swift_client.logger
-    if val:
-        for handler in swift_logger.handlers:
-            if isinstance(handler, logging.StreamHandler):
-                # Already present
-                return
-        swift_logger.addHandler(logging.StreamHandler())
-        swift_logger.setLevel(logging.DEBUG)
-    else:
-        for handler in swift_logger.handlers:
-            if isinstance(handler, logging.StreamHandler):
-                swift_logger.removeHandler(handler)
 
 
 def get_encoding():

awx/lib/site-packages/pyrax/autoscale.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2013 Rackspace
+# Copyright (c)2013 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -17,6 +17,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import base64
+
 import pyrax
 from pyrax.client import BaseClient
 from pyrax.cloudloadbalancers import CloudLoadBalancer
@@ -441,9 +443,9 @@ class ScalingGroupManager(BaseManager):
         largs = scaling_group.launchConfiguration.get("args", {})
         srv_args = largs.get("server", {})
         lb_args = largs.get("loadBalancers", {})
-        flav = "%s" % flavor or srv_args.get("flavorRef")
+        flav = flavor or srv_args.get("flavorRef")
-        dconf = disk_config or srv_args.get("OS-DCF:diskConfig")
+        dconf = disk_config or srv_args.get("OS-DCF:diskConfig", "AUTO")
-        pers = personality or srv_args.get("personality")
+        pers = personality or srv_args.get("personality", [])
         body = {"type": "launch_server",
                 "args": {
                     "server": {
@@ -451,13 +453,14 @@ class ScalingGroupManager(BaseManager):
                         "imageRef": image or srv_args.get("imageRef"),
                         "flavorRef": flav,
                         "OS-DCF:diskConfig": dconf,
-                        "personality": pers,
                         "networks": networks or srv_args.get("networks"),
                         "metadata": metadata or srv_args.get("metadata"),
                     },
                     "loadBalancers": load_balancers or lb_args,
                 },
             }
+        if pers:
+            body["args"]["server"]["personality"] = pers
         key_name = key_name or srv_args.get("key_name")
         if key_name:
             body["args"]["server"] = key_name
@@ -765,6 +768,10 @@ class ScalingGroupManager(BaseManager):
             metadata = {}
         if personality is None:
             personality = []
+        else:
+            for file in personality:
+                if "contents" in file:
+                    file["contents"] = base64.b64encode(file["contents"])
         if scaling_policies is None:
             scaling_policies = []
         group_config = self._create_group_config_body(name, cooldown,

awx/lib/site-packages/pyrax/cf_wrapper/client.py

@@ -1,6 +1,23 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+# Copyright (c)2012 Rackspace US, Inc.
+
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import logging
 import datetime
 from functools import wraps
 import hashlib
@@ -44,6 +61,7 @@ CONNECTION_TIMEOUT = 20
 CONNECTION_RETRIES = 5
 AUTH_ATTEMPTS = 2
 MAX_BULK_DELETE = 10000
+DEFAULT_CHUNKSIZE = 65536
 
 no_such_container_pattern = re.compile(
         r"Container (?:GET|HEAD) failed: .+/(.+) 404")
@@ -60,6 +78,17 @@ def _close_swiftclient_conn(conn):
         pass
 
 
+def plug_hole_in_swiftclient_auth(clt, url):
+    """
+    This is necessary because swiftclient has an issue when a token expires and
+    it needs to re-authenticate against Rackspace auth. It is a temporary
+    workaround until we can fix swiftclient.
+    """
+    conn = clt.connection
+    conn.token = clt.identity.token
+    conn.url = url
+
+
 def handle_swiftclient_exception(fnc):
     @wraps(fnc)
     def _wrapped(self, *args, **kwargs):
@@ -79,9 +108,9 @@ def handle_swiftclient_exception(fnc):
                         # Assume it is an auth failure. Re-auth and retry.
                         # NOTE: This is a hack to get around an apparent bug
                         # in python-swiftclient when using Rackspace auth.
-                        pyrax.authenticate(connect=False)
+                        self.identity.authenticate(connect=False)
-                        if pyrax.identity.authenticated:
+                        if self.identity.authenticated:
-                            pyrax.plug_hole_in_swiftclient_auth(self, clt_url)
+                            self.plug_hole_in_swiftclient_auth(self, clt_url)
                         continue
                 elif e.http_status == 404:
                     bad_container = no_such_container_pattern.search(str_error)
@@ -102,6 +131,16 @@ def handle_swiftclient_exception(fnc):
     return _wrapped
 
 
+def ensure_cdn(fnc):
+    @wraps(fnc)
+    def _wrapped(self, *args, **kwargs):
+        if not self.connection.cdn_connection:
+            raise exc.NotCDNEnabled("This service does not support "
+                    "CDN-enabled containers.")
+        return fnc(self, *args, **kwargs)
+    return _wrapped
+
+
 def _convert_head_object_last_modified_to_local(lm_str):
     # Need to convert last modified time to a datetime object.
     # Times are returned in default locale format, so we need to read
@@ -119,18 +158,32 @@ def _convert_head_object_last_modified_to_local(lm_str):
 
 
 def _convert_list_last_modified_to_local(attdict):
-    if 'last_modified' in attdict:
+    if "last_modified" in attdict:
         attdict = attdict.copy()
-        list_date_format_with_tz = LIST_DATE_FORMAT + ' %Z'
+        list_date_format_with_tz = LIST_DATE_FORMAT + " %Z"
-        last_modified_utc = attdict['last_modified'] + ' UTC'
+        last_modified_utc = attdict["last_modified"] + " UTC"
         tm_tuple = time.strptime(last_modified_utc,
                                  list_date_format_with_tz)
         dttm = datetime.datetime.fromtimestamp(time.mktime(tm_tuple))
-        attdict['last_modified'] = dttm.strftime(DATE_FORMAT)
 
+        dttm_with_micros = datetime.datetime.strptime(last_modified_utc,
+                list_date_format_with_tz)
+        # Round the date *up* in seconds, to match the last modified time
+        # in head requests
+        # https://review.openstack.org/#/c/55488/
+        if dttm_with_micros.microsecond > 0:
+            dttm += datetime.timedelta(seconds=1)
+        attdict["last_modified"] = dttm.strftime(DATE_FORMAT)
     return attdict
 
 
+def _quote(val):
+    if isinstance(val, six.text_type):
+        val = val.encode("utf-8")
+    return urllib.quote(val)
+
+
+
 class CFClient(object):
     """
     Wraps the calls to swiftclient with objects representing Containers
@@ -139,11 +192,6 @@ class CFClient(object):
     These classes allow a developer to work with regular Python objects
     instead of calling functions that return primitive types.
     """
-    # Constants used in metadata headers
-    account_meta_prefix = "X-Account-Meta-"
-    container_meta_prefix = "X-Container-Meta-"
-    object_meta_prefix = "X-Object-Meta-"
-    cdn_meta_prefix = "X-Cdn-"
     # Defaults for CDN
     cdn_enabled = False
     default_cdn_ttl = 86400
@@ -175,6 +223,24 @@ class CFClient(object):
                 http_log_debug=http_log_debug)
 
 
+    # Constants used in metadata headers
+    @property
+    def account_meta_prefix(self):
+        return "X-Account-Meta-"
+
+    @property
+    def container_meta_prefix(self):
+        return "X-Container-Meta-"
+
+    @property
+    def object_meta_prefix(self):
+        return "X-Object-Meta-"
+
+    @property
+    def cdn_meta_prefix(self):
+        return "X-Cdn-"
+
+
     def _make_connections(self, auth_endpoint, username, api_key, password,
             tenant_name=None, preauthurl=None, preauthtoken=None,
             auth_version="2", os_options=None, verify_ssl=True,
@@ -210,12 +276,14 @@ class CFClient(object):
 
 
     @handle_swiftclient_exception
-    def get_account_metadata(self):
+    def get_account_metadata(self, prefix=None):
         headers = self.connection.head_account()
-        prfx = self.account_meta_prefix.lower()
+        if prefix is None:
+            prefix = self.account_meta_prefix
+        prefix = prefix.lower()
         ret = {}
         for hkey, hval in headers.iteritems():
-            if hkey.lower().startswith(prfx):
+            if hkey.lower().startswith(prefix):
                 ret[hkey] = hval
         return ret
 
@@ -336,11 +404,7 @@ class CFClient(object):
         specified number of seconds.
         """
         meta = {"X-Delete-After": str(seconds)}
-        self.set_object_metadata(cont, obj, meta, prefix="")
+        self.set_object_metadata(cont, obj, meta, prefix="", clear=True)
-#        cname = self._resolve_name(cont)
-#        oname = self._resolve_name(obj)
-#        self.connection.post_object(cname, oname, headers=headers,
-#                response_dict=extra_info)
 
 
     @handle_swiftclient_exception
@@ -349,7 +413,8 @@ class CFClient(object):
         cname = self._resolve_name(container)
         headers = self.connection.head_container(cname)
         if prefix is None:
-            prefix = self.container_meta_prefix.lower()
+            prefix = self.container_meta_prefix
+        prefix = prefix.lower()
         ret = {}
         for hkey, hval in headers.iteritems():
             if hkey.lower().startswith(prefix):
@@ -394,19 +459,23 @@ class CFClient(object):
 
     @handle_swiftclient_exception
     def remove_container_metadata_key(self, container, key,
-            extra_info=None):
+            prefix=None, extra_info=None):
         """
         Removes the specified key from the container's metadata. If the key
         does not exist in the metadata, nothing is done.
         """
+        if prefix is None:
+            prefix = self.container_meta_prefix
+        prefix = prefix.lower()
         meta_dict = {key: ""}
         # Add the metadata prefix, if needed.
-        massaged = self._massage_metakeys(meta_dict, self.container_meta_prefix)
+        massaged = self._massage_metakeys(meta_dict, prefix)
         cname = self._resolve_name(container)
         self.connection.post_container(cname, massaged,
                 response_dict=extra_info)
 
 
+    @ensure_cdn
     @handle_swiftclient_exception
     def get_container_cdn_metadata(self, container):
         """
@@ -421,6 +490,7 @@ class CFClient(object):
         return dict(headers)
 
 
+    @ensure_cdn
     @handle_swiftclient_exception
     def set_container_cdn_metadata(self, container, metadata):
         """
@@ -449,15 +519,17 @@ class CFClient(object):
 
 
     @handle_swiftclient_exception
-    def get_object_metadata(self, container, obj):
+    def get_object_metadata(self, container, obj, prefix=None):
         """Retrieves any metadata for the specified object."""
+        if prefix is None:
+            prefix = self.object_meta_prefix
         cname = self._resolve_name(container)
         oname = self._resolve_name(obj)
         headers = self.connection.head_object(cname, oname)
-        prfx = self.object_meta_prefix.lower()
+        prefix = prefix.lower()
         ret = {}
         for hkey, hval in headers.iteritems():
-            if hkey.lower().startswith(prfx):
+            if hkey.lower().startswith(prefix):
                 ret[hkey] = hval
         return ret
 
@@ -493,8 +565,8 @@ class CFClient(object):
         # whereas for containers you need to set the values to an empty
         # string to delete them.
         if not clear:
-            obj_meta = self.get_object_metadata(cname, oname)
+            obj_meta = self.get_object_metadata(cname, oname, prefix=prefix)
-            new_meta = self._massage_metakeys(obj_meta, self.object_meta_prefix)
+            new_meta = self._massage_metakeys(obj_meta, prefix)
         utils.case_insensitive_update(new_meta, massaged)
         # Remove any empty values, since the object metadata API will
         # store them.
@@ -509,12 +581,12 @@ class CFClient(object):
 
 
     @handle_swiftclient_exception
-    def remove_object_metadata_key(self, container, obj, key):
+    def remove_object_metadata_key(self, container, obj, key, prefix=None):
         """
-        Removes the specified key from the storage object's metadata. If the key
+        Removes the specified key from the storage object's metadata. If the
-        does not exist in the metadata, nothing is done.
+        key does not exist in the metadata, nothing is done.
         """
-        self.set_object_metadata(container, obj, {key: ""})
+        self.set_object_metadata(container, obj, {key: ""}, prefix=prefix)
 
 
     @handle_swiftclient_exception
@@ -629,33 +701,50 @@ class CFClient(object):
     @handle_swiftclient_exception
     def store_object(self, container, obj_name, data, content_type=None,
             etag=None, content_encoding=None, ttl=None, return_none=False,
-            extra_info=None):
+            chunk_size=None, headers=None, extra_info=None):
         """
         Creates a new object in the specified container, and populates it with
         the given data. A StorageObject reference to the uploaded file
         will be returned, unless 'return_none' is set to True.
 
+        'chunk_size' represents the number of bytes of data to write; it
+        defaults to 65536. It is used only if the the 'data' parameter is an
+        object with a 'read' method; otherwise, it is ignored.
+
+        If you wish to specify additional headers to be passed to the PUT
+        request, pass them as a dict in the 'headers' parameter. It is the
+        developer's responsibility to ensure that any headers are valid; pyrax
+        does no checking.
+
         'extra_info' is an optional dictionary which will be
         populated with 'status', 'reason', and 'headers' keys from the
         underlying swiftclient call.
         """
         cont = self.get_container(container)
-        headers = {}
+        if headers is None:
+            headers = {}
         if content_encoding is not None:
             headers["Content-Encoding"] = content_encoding
         if ttl is not None:
             headers["X-Delete-After"] = ttl
-        with utils.SelfDeletingTempfile() as tmp:
+        if chunk_size and hasattr(data, "read"):
-            with open(tmp, "wb") as tmpfile:
+            # Chunked file-like object
-                try:
+            self.connection.put_object(cont.name, obj_name, contents=data,
-                    tmpfile.write(data)
+                    content_type=content_type, etag=etag, headers=headers,
-                except UnicodeEncodeError:
+                    chunk_size=chunk_size, response_dict=extra_info)
-                    udata = data.encode("utf-8")
+        else:
-                    tmpfile.write(udata)
+            with utils.SelfDeletingTempfile() as tmp:
-            with open(tmp, "rb") as tmpfile:
+                with open(tmp, "wb") as tmpfile:
-                self.connection.put_object(cont.name, obj_name,
+                    try:
-                        contents=tmpfile, content_type=content_type, etag=etag,
+                        tmpfile.write(data)
-                        headers=headers, response_dict=extra_info)
+                    except UnicodeEncodeError:
+                        udata = data.encode("utf-8")
+                        tmpfile.write(udata)
+                with open(tmp, "rb") as tmpfile:
+                    self.connection.put_object(cont.name, obj_name,
+                            contents=tmpfile, content_type=content_type,
+                            etag=etag, headers=headers, chunk_size=chunk_size,
+                            response_dict=extra_info)
         if return_none:
             return None
         else:
@@ -721,7 +810,7 @@ class CFClient(object):
     def upload_file(self, container, file_or_path, obj_name=None,
             content_type=None, etag=None, return_none=False,
             content_encoding=None, ttl=None, extra_info=None,
-            content_length=None):
+            content_length=None, headers=None):
         """
         Uploads the specified file to the container. If no name is supplied,
         the file's name will be used. Either a file path or an open file-like
@@ -734,6 +823,11 @@ class CFClient(object):
 
         If the size of the file is known, it can be passed as `content_length`.
 
+        If you wish to specify additional headers to be passed to the PUT
+        request, pass them as a dict in the 'headers' parameter. It is the
+        developer's responsibility to ensure that any headers are valid; pyrax
+        does no checking.
+
         If you wish for the object to be temporary, specify the time it should
         be stored in seconds in the `ttl` parameter. If this is specified, the
         object will be deleted after that number of seconds.
@@ -806,7 +900,8 @@ class CFClient(object):
             raise InvalidUploadID("No filename provided and/or it cannot be "
                     "inferred from context")
 
-        headers = {}
+        if headers is None:
+            headers = {}
         if content_encoding is not None:
             headers["Content-Encoding"] = content_encoding
         if ttl is not None:
@@ -886,7 +981,8 @@ class CFClient(object):
 
 
     def sync_folder_to_container(self, folder_path, container, delete=False,
-            include_hidden=False, ignore=None, ignore_timestamps=False):
+            include_hidden=False, ignore=None, ignore_timestamps=False,
+            object_prefix="", verbose=False):
         """
         Compares the contents of the specified folder, and checks to make sure
         that the corresponding object is present in the specified container. If
@@ -909,22 +1005,43 @@ class CFClient(object):
         file names, and any names that match any of the 'ignore' patterns will
         not be uploaded. The patterns should be standard *nix-style shell
         patterns; e.g., '*pyc' will ignore all files ending in 'pyc', such as
-        'program.pyc' and 'abcpyc'.  """
+        'program.pyc' and 'abcpyc'.
+
+        If `object_prefix` is set it will be appended to the object name when
+        it is checked and uploaded to the container. For example, if you use
+        sync_folder_to_container("folderToSync/", myContainer,
+            object_prefix="imgFolder") it will upload the files to the
+        container/imgFolder/... instead of just container/...
+
+        Set `verbose` to True to make it print what is going on. It will
+        show which files are being uploaded and which ones are not and why.
+        """
         cont = self.get_container(container)
         self._local_files = []
+        # Load a list of all the remote objects so we don't have to keep
+        # hitting the service
+        if verbose:
+            log = logging.getLogger("pyrax")
+            log.info("Loading remote object list (prefix=%s)", object_prefix)
+        data = cont.get_objects(prefix=object_prefix, full_listing=True)
+        self._remote_files = dict((d.name, d) for d in data)
         self._sync_folder_to_container(folder_path, cont, prefix="",
                 delete=delete, include_hidden=include_hidden, ignore=ignore,
-                ignore_timestamps=ignore_timestamps)
+                ignore_timestamps=ignore_timestamps,
+                object_prefix=object_prefix, verbose=verbose)
+        # Unset the _remote_files
+        self._remote_files = None
 
 
     def _sync_folder_to_container(self, folder_path, cont, prefix, delete,
-            include_hidden, ignore, ignore_timestamps):
+            include_hidden, ignore, ignore_timestamps, object_prefix, verbose):
         """
         This is the internal method that is called recursively to handle
         nested folder structures.
         """
         fnames = os.listdir(folder_path)
         ignore = utils.coerce_string_to_list(ignore)
+        log = logging.getLogger("pyrax")
         if not include_hidden:
             ignore.append(".*")
         for fname in fnames:
@@ -937,17 +1054,20 @@ class CFClient(object):
                     subprefix = "%s/%s" % (prefix, subprefix)
                 self._sync_folder_to_container(pth, cont, prefix=subprefix,
                         delete=delete, include_hidden=include_hidden,
-                        ignore=ignore, ignore_timestamps=ignore_timestamps)
+                        ignore=ignore, ignore_timestamps=ignore_timestamps,
+                        object_prefix=object_prefix, verbose=verbose)
                 continue
-            self._local_files.append(os.path.join(prefix, fname))
+            self._local_files.append(os.path.join(object_prefix, prefix, fname))
             local_etag = utils.get_checksum(pth)
             fullname = fname
+            fullname_with_prefix = "%s/%s" % (object_prefix, fname)
             if prefix:
                 fullname = "%s/%s" % (prefix, fname)
+                fullname_with_prefix = "%s/%s/%s" % (object_prefix, prefix, fname)
             try:
-                obj = cont.get_object(fullname)
+                obj = self._remote_files[fullname_with_prefix]
                 obj_etag = obj.etag
-            except exc.NoSuchObject:
+            except KeyError:
                 obj = None
                 obj_etag = None
             if local_etag != obj_etag:
@@ -961,19 +1081,29 @@ class CFClient(object):
                     local_mod_str = local_mod.isoformat()
                     if obj_time_str >= local_mod_str:
                         # Remote object is newer
+                        if verbose:
+                            log.info("%s NOT UPLOADED because remote object is "
+                                    "newer", fullname)
                         continue
-                cont.upload_file(pth, obj_name=fullname, etag=local_etag,
+                cont.upload_file(pth, obj_name=fullname_with_prefix,
-                        return_none=True)
+                    etag=local_etag, return_none=True)
+                if verbose:
+                    log.info("%s UPLOADED", fullname)
+            else:
+                if verbose:
+                    log.info("%s NOT UPLOADED because it already exists",
+                            fullname)
         if delete and not prefix:
-            self._delete_objects_not_in_list(cont)
+            self._delete_objects_not_in_list(cont, object_prefix)
 
 
-    def _delete_objects_not_in_list(self, cont):
+    def _delete_objects_not_in_list(self, cont, object_prefix=""):
         """
         Finds all the objects in the specified container that are not present
         in the self._local_files list, and deletes them.
         """
-        objnames = set(cont.get_object_names(full_listing=True))
+        objnames = set(cont.get_object_names(prefix=object_prefix,
+                full_listing=True))
         localnames = set(self._local_files)
         to_delete = list(objnames.difference(localnames))
         # We don't need to wait around for this to complete. Store the thread
@@ -1067,6 +1197,52 @@ class CFClient(object):
         return ret
 
 
+    def fetch_dlo(self, cont, name, chunk_size=None):
+        """
+        Returns a list of 2-tuples in the form of (object_name,
+        fetch_generator) representing the components of a multi-part DLO
+        (Dynamic Large Object).  Each fetch_generator object can be interated
+        to retrieve its contents.
+
+        This is useful when transferring a DLO from one object storage system
+        to another. Examples would be copying DLOs from one region of a
+        provider to another, or copying a DLO from one provider to another.
+        """
+        if chunk_size is None:
+            chunk_size = DEFAULT_CHUNKSIZE
+
+        class FetchChunker(object):
+            """
+            Class that takes the generator objects returned by a chunked
+            fetch_object() call and wraps them to behave as file-like objects for
+            uploading.
+            """
+            def __init__(self, gen, verbose=False):
+                self.gen = gen
+                self.verbose = verbose
+                self.processed = 0
+                self.interval = 0
+
+            def read(self, size=None):
+                self.interval += 1
+                if self.verbose:
+                    if self.interval > 1024:
+                        self.interval = 0
+                        logit(".")
+                ret = self.gen.next()
+                self.processed += len(ret)
+                return ret
+
+        parts = self.get_container_objects(cont, prefix=name)
+        fetches = [(part.name, self.fetch_object(cont, part.name,
+                    chunk_size=chunk_size))
+                for part in parts
+                if part.name != name]
+        job = [(fetch[0], FetchChunker(fetch[1], verbose=False))
+                for fetch in fetches]
+        return job
+
+
     @handle_swiftclient_exception
     def download_object(self, container, obj, directory, structure=True):
         """
@@ -1096,7 +1272,8 @@ class CFClient(object):
 
     @handle_swiftclient_exception
     def get_all_containers(self, limit=None, marker=None, **parms):
-        hdrs, conts = self.connection.get_container("")
+        hdrs, conts = self.connection.get_container("", limit=limit,
+                marker=marker)
         ret = [Container(self, name=cont["name"], object_count=cont["count"],
                 total_bytes=cont["bytes"]) for cont in conts]
         return ret
@@ -1123,6 +1300,7 @@ class CFClient(object):
                     total_bytes=hdrs.get("x-container-bytes-used"))
             self._container_cache[cname] = cont
         return cont
+    get = get_container
 
 
     @handle_swiftclient_exception
@@ -1145,6 +1323,7 @@ class CFClient(object):
                               attdict=_convert_list_last_modified_to_local(obj))
                 for obj in objs
                 if "name" in obj]
+    list_container_objects = get_container_objects
 
 
     @handle_swiftclient_exception
@@ -1156,24 +1335,26 @@ class CFClient(object):
                 full_listing=full_listing)
         cont = self.get_container(cname)
         return [obj["name"] for obj in objs]
+    list_container_object_names = get_container_object_names
 
 
     @handle_swiftclient_exception
     def list_container_subdirs(self, container, marker=None, limit=None,
             prefix=None, delimiter=None, full_listing=False):
         """
-        Return a list of StorageObjects representing the pseudo-subdirectories
+        Returns a list of StorageObjects representing the pseudo-subdirectories
         in the specified container. You can use the marker and limit params to
-        handle pagination, and the prefix and delimiter params to filter the
+        handle pagination, and the prefix param to filter the objects returned.
-        objects returned.
+        The 'delimiter' parameter is ignored, as the only meaningful value is
+        '/'.
         """
         cname = self._resolve_name(container)
         hdrs, objs = self.connection.get_container(cname, marker=marker,
-                limit=limit, prefix=prefix, delimiter=delimiter,
+                limit=limit, prefix=prefix, delimiter="/",
                 full_listing=full_listing)
         cont = self.get_container(cname)
         return [StorageObject(self, container=cont, attdict=obj) for obj in objs
-                if obj.get("content_type") == "application/directory"]
+                if "subdir" in obj]
 
 
     @handle_swiftclient_exception
@@ -1189,17 +1370,21 @@ class CFClient(object):
     @handle_swiftclient_exception
     def list(self, limit=None, marker=None, **parms):
         """Returns a list of all container objects."""
-        hdrs, conts = self.connection.get_container("")
+        hdrs, conts = self.connection.get_container("", limit=limit,
+                marker=marker)
         ret = [self.get_container(cont["name"]) for cont in conts]
         return ret
+    get_all_containers = list
 
 
     @handle_swiftclient_exception
     def list_containers(self, limit=None, marker=None, **parms):
         """Returns a list of all container names as strings."""
-        hdrs, conts = self.connection.get_container("")
+        hdrs, conts = self.connection.get_container("", limit=limit,
+                marker=marker)
         ret = [cont["name"] for cont in conts]
         return ret
+    list_container_names = list_containers
 
 
     @handle_swiftclient_exception
@@ -1212,10 +1397,12 @@ class CFClient(object):
             count - the number of objects in the container
             bytes - the total bytes in the container
         """
-        hdrs, conts = self.connection.get_container("")
+        hdrs, conts = self.connection.get_container("", limit=limit,
+                marker=marker)
         return conts
 
 
+    @ensure_cdn
     @handle_swiftclient_exception
     def list_public_containers(self):
         """Returns a list of all CDN-enabled containers."""
@@ -1240,6 +1427,7 @@ class CFClient(object):
         return self._cdn_set_access(container, None, False)
 
 
+    @ensure_cdn
     def _cdn_set_access(self, container, ttl, enabled):
         """Used to enable or disable CDN access on a container."""
         if ttl is None:
@@ -1273,6 +1461,7 @@ class CFClient(object):
         cont.cdn_log_retention = enabled
 
 
+    @ensure_cdn
     def _set_cdn_log_retention(self, container, enabled):
         """This does the actual call to the Cloud Files API."""
         hdrs = {"X-Log-Retention": "%s" % enabled}
@@ -1324,6 +1513,7 @@ class CFClient(object):
         return self.set_container_metadata(container, hdr, clear=False)
 
 
+    @ensure_cdn
     @handle_swiftclient_exception
     def purge_cdn_object(self, container, name, email_addresses=None):
         ct = self.get_container(container)
@@ -1413,12 +1603,7 @@ class Connection(_swift_client.Connection):
 
         Taken directly from the cloudfiles library and modified for use here.
         """
-        def quote(val):
+        pth = "/".join([_quote(elem) for elem in path])
-            if isinstance(val, six.text_type):
-                val = val.encode("utf-8")
-            return urllib.quote(val)
-
-        pth = "/".join([quote(elem) for elem in path])
         uri_path = urlparse.urlparse(self.uri).path
         path = "%s/%s" % (uri_path.rstrip("/"), pth)
         headers = {"Content-Length": str(len(data)),
@@ -1440,8 +1625,8 @@ class Connection(_swift_client.Connection):
                 response = None
             if response:
                 if response.status == 401:
-                    pyrax.identity.authenticate()
+                    self.identity.authenticate()
-                    headers["X-Auth-Token"] = pyrax.identity.token
+                    headers["X-Auth-Token"] = self.identity.token
                 else:
                     break
             attempt += 1
@@ -1532,14 +1717,14 @@ class BulkDeleter(threading.Thread):
         cname = client._resolve_name(container)
         parsed, conn = client.connection.http_connection()
         method = "DELETE"
-        headers = {"X-Auth-Token": pyrax.identity.token,
+        headers = {"X-Auth-Token": self.client.identity.token,
                 "Content-type": "text/plain",
                 }
         while object_names:
             this_batch, object_names = (object_names[:MAX_BULK_DELETE],
                     object_names[MAX_BULK_DELETE:])
             obj_paths = ("%s/%s" % (cname, nm) for nm in this_batch)
-            body = "\n".join(obj_paths)
+            body = _quote("\n".join(obj_paths))
             pth = "%s/?bulk-delete=1" % parsed.path
             conn.request(method, pth, body, headers)
             resp = conn.getresponse()
@@ -1547,7 +1732,7 @@ class BulkDeleter(threading.Thread):
             reason = resp.reason
             resp_body = resp.read()
             for resp_line in resp_body.splitlines():
-                if not resp_line:
+                if not resp_line.strip():
                     continue
                 resp_key, val = resp_line.split(":", 1)
                 result_key = res_keys.get(resp_key)

awx/lib/site-packages/pyrax/cf_wrapper/container.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -56,6 +56,10 @@ class Container(object):
 
     def _fetch_cdn_data(self):
         """Fetches the object's CDN data from the CDN service"""
+        if not self.client.cdn_enabled:
+            # Not CDN enabled; set the defaults.
+            self._set_cdn_defaults()
+            return
         response = self.client.connection.cdn_request("HEAD", [self.name])
         if 200 <= response.status < 300:
             # Set defaults in case not all headers are present.
@@ -95,6 +99,7 @@ class Container(object):
                 limit=limit, prefix=prefix, delimiter=delimiter,
                 full_listing=full_listing)
         return objs
+    list = get_objects
 
 
     def get_object(self, name, cached=True):
@@ -113,6 +118,7 @@ class Container(object):
             ret = self.client.get_object(self, name)
             self._object_cache[name] = ret
         return ret
+    get = get_object
 
 
     def get_object_names(self, marker=None, limit=None, prefix=None,
@@ -124,6 +130,7 @@ class Container(object):
         return self.client.get_container_object_names(self.name, marker=marker,
                 limit=limit, prefix=prefix, delimiter=delimiter,
                 full_listing=full_listing)
+    list_object_names = get_object_names
 
 
     def list_subdirs(self, marker=None, limit=None, prefix=None, delimiter=None,
@@ -244,11 +251,11 @@ class Container(object):
                 structure=structure)
 
 
-    def get_metadata(self):
+    def get_metadata(self, prefix=None):
         """
         Returns a dictionary containing the metadata for the container.
         """
-        return self.client.get_container_metadata(self)
+        return self.client.get_container_metadata(self, prefix=prefix)
 
 
     def set_metadata(self, metadata, clear=False, prefix=None):
@@ -273,12 +280,13 @@ class Container(object):
                 prefix=prefix)
 
 
-    def remove_metadata_key(self, key):
+    def remove_metadata_key(self, key, prefix=None):
         """
         Removes the specified key from the container's metadata. If the key
         does not exist in the metadata, nothing is done.
         """
-        return self.client.remove_container_metadata_key(self, key)
+        return self.client.remove_container_metadata_key(self, key,
+                prefix=prefix)
 
 
     def set_web_index_page(self, page):

awx/lib/site-packages/pyrax/cf_wrapper/storage_object.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -106,9 +106,10 @@ class StorageObject(object):
                 name=self.name, email_addresses=email_addresses)
 
 
-    def get_metadata(self):
+    def get_metadata(self, prefix=None):
         """Returns this object's metadata."""
-        return self.client.get_object_metadata(self.container, self)
+        return self.client.get_object_metadata(self.container, self,
+                prefix=prefix)
 
 
     def set_metadata(self, metadata, clear=False, prefix=None):
@@ -119,12 +120,13 @@ class StorageObject(object):
                 clear=clear, prefix=prefix)
 
 
-    def remove_metadata_key(self, key):
+    def remove_metadata_key(self, key, prefix=None):
         """
         Removes the specified key from the storage object's metadata. If the
         key does not exist in the metadata, nothing is done.
         """
-        self.client.remove_object_metadata_key(self.container, self, key)
+        self.client.remove_object_metadata_key(self.container, self, key,
+                prefix=prefix)
 
 
     def copy(self, new_container, new_obj_name=None, extra_info=None):

awx/lib/site-packages/pyrax/client.py

@@ -1,7 +1,10 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
 # Copyright 2010 Jacob Kaplan-Moss
 # Copyright 2011 OpenStack LLC.
 # Copyright 2011 Piston Cloud Computing, Inc.
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -20,17 +23,30 @@
 OpenStack Client interface. Handles the REST calls and responses.
 """
 
+from __future__ import absolute_import
+
 import json
 import logging
 import requests
 import time
-import urllib
+import six.moves.urllib as urllib
-import urlparse
 
 import pyrax
 import pyrax.exceptions as exc
 
 
+def _safe_quote(val):
+    """
+    Unicode values will raise a KeyError, so catch those and encode in UTF-8.
+    """
+    SAFE_QUOTE_CHARS = "/.?&=,"
+    try:
+        ret = urllib.parse.quote(val, safe=SAFE_QUOTE_CHARS)
+    except KeyError:
+        ret = urllib.parse.quote(val.encode("utf-8"), safe=SAFE_QUOTE_CHARS)
+    return ret
+
+
 class BaseClient(object):
     """
     The base class for all pyrax clients.
@@ -40,10 +56,11 @@ class BaseClient(object):
     # Each client subclass should set their own name.
     name = "base"
 
-    def __init__(self, region_name=None, endpoint_type="publicURL",
+    def __init__(self, identity, region_name=None, endpoint_type=None,
             management_url=None, service_name=None, timings=False,
             verify_ssl=True, http_log_debug=False, timeout=None):
         self.version = "v1.1"
+        self.identity = identity
         self.region_name = region_name
         self.endpoint_type = endpoint_type
         self.service_name = service_name
@@ -114,7 +131,7 @@ class BaseClient(object):
 
     def unauthenticate(self):
         """Clears all of our authentication information."""
-        pyrax.identity.unauthenticate()
+        self.identity.unauthenticate()
 
 
     def get_timings(self):
@@ -158,6 +175,11 @@ class BaseClient(object):
         kwargs.setdefault("headers", kwargs.get("headers", {}))
         kwargs["headers"]["User-Agent"] = self.user_agent
         kwargs["headers"]["Accept"] = "application/json"
+        if ("body" in kwargs) or ("data" in kwargs):
+            if "Content-Type" not in kwargs["headers"]:
+                kwargs["headers"]["Content-Type"] = "application/json"
+            elif kwargs["headers"]["Content-Type"] is None:
+                del kwargs["headers"]["Content-Type"]
         # Allow subclasses to add their own headers
         self._add_custom_headers(kwargs["headers"])
         resp, body = pyrax.http.request(method, uri, *args, **kwargs)
@@ -181,7 +203,7 @@ class BaseClient(object):
         the request after authenticating if the initial request returned
         and Unauthorized exception.
         """
-        id_svc = pyrax.identity
+        id_svc = self.identity
         if not all((self.management_url, id_svc.token, id_svc.tenant_id)):
             id_svc.authenticate()
 
@@ -191,16 +213,15 @@ class BaseClient(object):
             raise exc.ServiceNotAvailable("The '%s' service is not available."
                     % self)
         if uri.startswith("http"):
-            parsed = list(urlparse.urlparse(uri))
+            parsed = list(urllib.parse.urlparse(uri))
             for pos, item in enumerate(parsed):
                 if pos < 2:
                     # Don't escape the scheme or netloc
                     continue
-                parsed[pos] = urllib.quote(parsed[pos], safe="/.?&=,")
+                parsed[pos] = _safe_quote(parsed[pos])
-            safe_uri = urlparse.urlunparse(parsed)
+            safe_uri = urllib.parse.urlunparse(parsed)
         else:
-            safe_uri = "%s%s" % (self.management_url,
+            safe_uri = "%s%s" % (self.management_url, _safe_quote(uri))
-                    urllib.quote(uri, safe="/.?&=,"))
         # Perform the request once. If we get a 401 back then it
         # might be because the auth token expired, so try to
         # re-authenticate and try again. If it still fails, bail.
@@ -258,7 +279,7 @@ class BaseClient(object):
         to modify this method. Please post your findings on GitHub so that
         others can benefit.
         """
-        return pyrax.identity.authenticate()
+        return self.identity.authenticate()
 
 
     @property
@@ -267,4 +288,4 @@ class BaseClient(object):
         The older parts of this code used 'projectid'; this wraps that
         reference.
         """
-        return pyrax.identity.tenant_id
+        return self.identity.tenant_id

awx/lib/site-packages/pyrax/cloudblockstorage.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -27,6 +27,7 @@ from pyrax.client import BaseClient
 import pyrax.exceptions as exc
 from pyrax.manager import BaseManager
 from pyrax.resource import BaseResource
+import pyrax.utils as utils
 
 
 MIN_SIZE = 100
@@ -129,7 +130,9 @@ class CloudBlockStorageVolume(BaseResource):
     def __init__(self, *args, **kwargs):
         super(CloudBlockStorageVolume, self).__init__(*args, **kwargs)
         region = self.manager.api.region_name
-        self._nova_volumes = pyrax.connect_to_cloudservers(region).volumes
+        context = self.manager.api.identity
+        cs = pyrax.connect_to_cloudservers(region=region, context=context)
+        self._nova_volumes = cs.volumes
 
 
     def attach_to_instance(self, instance, mountpoint):
@@ -286,6 +289,24 @@ class CloudBlockStorageManager(BaseManager):
                 raise
 
 
+    def update(self, volume, display_name=None, display_description=None):
+        """
+        Update the specified values on the specified volume. You may specify
+        one or more values to update.
+        """
+        uri = "/%s/%s" % (self.uri_base, utils.get_id(volume))
+        param_dict = {}
+        if display_name:
+            param_dict["display_name"] = display_name
+        if display_description:
+            param_dict["display_description"] = display_description
+        if not param_dict:
+            # Nothing to do!
+            return
+        body = {"volume": param_dict}
+        resp, resp_body = self.api.method_put(uri, body=body)
+
+
     def list_snapshots(self):
         """
         Pass-through method to allow the list_snapshots() call to be made
@@ -354,6 +375,24 @@ class CloudBlockStorageSnapshotManager(BaseManager):
         return snap
 
 
+    def update(self, snapshot, display_name=None, display_description=None):
+        """
+        Update the specified values on the specified snapshot. You may specify
+        one or more values to update.
+        """
+        uri = "/%s/%s" % (self.uri_base, utils.get_id(snapshot))
+        param_dict = {}
+        if display_name:
+            param_dict["display_name"] = display_name
+        if display_description:
+            param_dict["display_description"] = display_description
+        if not param_dict:
+            # Nothing to do!
+            return
+        body = {"snapshot": param_dict}
+        resp, resp_body = self.api.method_put(uri, body=body)
+
+
 class CloudBlockStorageClient(BaseClient):
     """
     This is the primary class for interacting with Cloud Block Storage.
@@ -404,6 +443,16 @@ class CloudBlockStorageClient(BaseClient):
         return volume.delete(force=force)
 
 
+    @assure_volume
+    def update(self, volume, display_name=None, display_description=None):
+        """
+        Update the specified values on the specified volume. You may specify
+        one or more values to update.
+        """
+        return self._manager.update(volume, display_name=display_name,
+                display_description=display_description)
+
+
     @assure_volume
     def create_snapshot(self, volume, name=None, description=None, force=False):
         """
@@ -416,7 +465,25 @@ class CloudBlockStorageClient(BaseClient):
                 description=description, force=force)
 
 
+    def get_snapshot(self, snapshot):
+        """
+        Returns the snapshot with the specified snapshot ID value.
+        """
+        return self._snapshot_manager.get(snapshot)
+
+
     @assure_snapshot
     def delete_snapshot(self, snapshot):
         """Deletes the snapshot."""
         return snapshot.delete()
+
+
+    def update_snapshot(self, snapshot, display_name=None,
+            display_description=None):
+        """
+        Update the specified values on the specified snapshot. You may specify
+        one or more values to update.
+        """
+        return self._snapshot_manager.update(snapshot,
+                display_name=display_name,
+                display_description=display_description)

awx/lib/site-packages/pyrax/clouddatabases.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #

awx/lib/site-packages/pyrax/clouddns.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -871,12 +871,14 @@ class CloudDNSManager(BaseManager):
         Takes a device and device type and returns the corresponding HREF link
         and service name for use with PTR record management.
         """
+        context = self.api.identity
+        region = self.api.region_name
         if device_type.lower().startswith("load"):
-            ep = pyrax._get_service_endpoint("load_balancer")
+            ep = pyrax._get_service_endpoint(context, "load_balancer", region)
             svc = "loadbalancers"
             svc_name = "cloudLoadBalancers"
         else:
-            ep = pyrax._get_service_endpoint("compute")
+            ep = pyrax._get_service_endpoint(context, "compute", region)
             svc = "servers"
             svc_name = "cloudServersOpenStack"
         href = "%s/%s/%s" % (ep, svc, utils.get_id(device))

awx/lib/site-packages/pyrax/cloudloadbalancers.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -85,9 +85,15 @@ class CloudLoadBalancer(BaseResource):
         return self.manager.get_usage(self, start=start, end=end)
 
 
+    def get_stats(self):
+        """
+        Return the stats for this loadbalancer
+        """
+        return self.manager.get_stats(self)
+
     def _add_details(self, info):
         """Override the base behavior to add Nodes, VirtualIPs, etc."""
-        for (key, val) in info.iteritems():
+        for (key, val) in six.iteritems(info):
             if key == "nodes":
                 val = [Node(parent=self, **nd) for nd in val]
             elif key == "sessionPersistence":
@@ -956,7 +962,7 @@ class CloudLoadBalancerManager(BaseManager):
         return body
 
 
-    def get_stats(self, loadbalancer):
+    def get_stats(self, loadbalancer=None):
         """
         Returns statistics for the given load balancer.
         """

awx/lib/site-packages/pyrax/cloudmonitoring.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2013 Rackspace
+# Copyright (c)2013 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -162,7 +162,7 @@ class CloudMonitorNotificationManager(BaseManager):
                 "details": details,
                 }
         resp, resp_body = self.api.method_post(uri, body=body)
-        return self.get(resp["x-object-id"])
+        return self.get(resp.headers["x-object-id"])
 
 
     def test_notification(self, notification=None, notification_type=None,
@@ -259,7 +259,7 @@ class CloudMonitorNotificationPlanManager(BaseManager):
             ok_state = utils.coerce_string_to_list(ok_state)
             body["ok_state"] = make_list_of_ids(ok_state)
         resp, resp_body = self.api.method_post(uri, body=body)
-        return self.get(resp["x-object-id"])
+        return self.get(resp.headers["x-object-id"])
 
 
 class CloudMonitorEntityManager(BaseManager):
@@ -380,9 +380,8 @@ class CloudMonitorEntityManager(BaseManager):
                     raise exc.InvalidMonitoringCheckDetails("Validation "
                             "failed. Error: '%s'." % dtls)
         else:
-            status = resp["status"]
+            if resp.status_code == 201:
-            if status == "201":
+                check_id = resp.headers["x-object-id"]
-                check_id = resp["x-object-id"]
                 return self.get_check(entity, check_id)
 
 
@@ -564,12 +563,11 @@ class CloudMonitorEntityManager(BaseManager):
         if metadata:
             body["metadata"] = metadata
         resp, resp_body = self.api.method_post(uri, body=body)
-
+        if resp.status_code == 201:
-        status = resp["status"]
+            alarm_id = resp.headers["x-object-id"]
-        if status == "201":
-            alarm_id = resp["x-object-id"]
             return self.get_alarm(entity, alarm_id)
 
+
     def update_alarm(self, entity, alarm, criteria=None, disabled=False,
             label=None, name=None, metadata=None):
         """
@@ -948,9 +946,8 @@ class CloudMonitorClient(BaseClient):
         resp = self._entity_manager.create(label=label, name=name, agent=agent,
                 ip_addresses=ip_addresses, metadata=metadata,
                 return_response=True)
-        status = resp["status"]
+        if resp.status_code == 201:
-        if status == "201":
+            ent_id = resp.headers["x-object-id"]
-            ent_id = resp["x-object-id"]
             return self.get_entity(ent_id)
 
 

awx/lib/site-packages/pyrax/cloudnetworks.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2013 Rackspace
+# Copyright (c)2013 Rackspace US, Inc.
 
 # All Rights Reserved.
 #

awx/lib/site-packages/pyrax/exceptions.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -245,6 +245,18 @@ class NetworkNotFound(PyraxException):
 class NetworkLabelNotUnique(PyraxException):
     pass
 
+class NoClientForService(PyraxException):
+    pass
+
+class NoEndpointForRegion(PyraxException):
+    pass
+
+class NoEndpointForService(PyraxException):
+    pass
+
+class NoContentSpecified(PyraxException):
+    pass
+
 class NoMoreResults(PyraxException):
     pass
 
@@ -254,6 +266,9 @@ class NoReloadError(PyraxException):
 class NoSSLTerminationConfiguration(PyraxException):
     pass
 
+class NoSuchClient(PyraxException):
+    pass
+
 class NoSuchContainer(PyraxException):
     pass
 
@@ -466,6 +481,8 @@ def from_response(response, body):
                 else:
                     message = error
                     details = None
+        else:
+            message = body
         return cls(code=status, message=message, details=details,
                    request_id=request_id)
     else:

awx/lib/site-packages/pyrax/fakes.py

@@ -12,10 +12,6 @@ from pyrax.autoscale import AutoScalePolicy
 from pyrax.autoscale import AutoScaleWebhook
 from pyrax.autoscale import ScalingGroup
 from pyrax.autoscale import ScalingGroupManager
-from pyrax.cf_wrapper.client import BulkDeleter
-from pyrax.cf_wrapper.client import FolderUploader
-from pyrax.cf_wrapper.container import Container
-from pyrax.cf_wrapper.storage_object import StorageObject
 from pyrax.client import BaseClient
 from pyrax.clouddatabases import CloudDatabaseClient
 from pyrax.clouddatabases import CloudDatabaseDatabaseManager
@@ -49,6 +45,13 @@ from pyrax.image import ImageClient
 from pyrax.image import ImageManager
 from pyrax.image import ImageMemberManager
 from pyrax.image import ImageTagManager
+from pyrax.object_storage import BulkDeleter
+from pyrax.object_storage import Container
+from pyrax.object_storage import ContainerManager
+from pyrax.object_storage import FolderUploader
+from pyrax.object_storage import StorageClient
+from pyrax.object_storage import StorageObject
+from pyrax.object_storage import StorageObjectManager
 from pyrax.queueing import Queue
 from pyrax.queueing import QueueClaim
 from pyrax.queueing import QueueMessage
@@ -56,6 +59,9 @@ from pyrax.queueing import QueueClient
 from pyrax.queueing import QueueManager
 
 import pyrax.exceptions as exc
+from pyrax.base_identity import BaseIdentity
+from pyrax.base_identity import Endpoint
+from pyrax.base_identity import Service
 from pyrax.identity.rax_identity import RaxIdentity
 from pyrax.identity.keystone_identity import KeystoneIdentity
 import pyrax.utils as utils
@@ -88,41 +94,72 @@ class FakeResponse(object):
         return "Line1\nLine2"
 
     def get(self, arg):
-        pass
+        return self.headers.get(arg)
 
     def json(self):
         return self.content
 
 
+class FakeIterator(utils.ResultsIterator):
+    def _init_methods(self):
+        pass
+
+
 class FakeClient(object):
     user_agent = "Fake"
     USER_AGENT = "Fake"
 
+    def __init__(self, *args, **kwargs):
+        self.identity = FakeIdentity()
+
+
+class FakeStorageClient(StorageClient):
+    def __init__(self, identity=None, *args, **kwargs):
+        if identity is None:
+            identity = FakeIdentity()
+        super(FakeStorageClient, self).__init__(identity, *args, **kwargs)
+
+    def create(self, name):
+        return FakeContainer(self._manager, {"name": name})
+
+
+class FakeContainerManager(ContainerManager):
+    def __init__(self, api=None, *args, **kwargs):
+        if api is None:
+            api = FakeStorageClient()
+        super(FakeContainerManager, self).__init__(api, *args, **kwargs)
+
 
 class FakeContainer(Container):
-    def _fetch_cdn_data(self):
+    def __init__(self, *args, **kwargs):
-        self._cdn_uri = None
+        super(FakeContainer, self).__init__(*args, **kwargs)
-        self._cdn_ttl = self.client.default_cdn_ttl
+        self.object_manager = FakeStorageObjectManager(self.manager.api,
-        self._cdn_ssl_uri = None
+                uri_base=self.name)
-        self._cdn_streaming_uri = None
+        self.object_manager._container = self
-        self._cdn_ios_uri = None
+
-        self._cdn_log_retention = False
+
+class FakeStorageObjectManager(StorageObjectManager):
+    def __init__(self, api=None, *args, **kwargs):
+        if api is None:
+            api = FakeStorageClient()
+        if "uri_base" not in kwargs:
+            kwargs["uri_base"] = utils.random_ascii()
+        super(FakeStorageObjectManager, self).__init__(api, *args, **kwargs)
 
 
 class FakeStorageObject(StorageObject):
-    def __init__(self, client, container, name=None, total_bytes=None,
+    def __init__(self, manager, name=None, total_bytes=None, content_type=None,
-            content_type=None, last_modified=None, etag=None, attdict=None):
+            last_modified=None, etag=None, attdict=None):
         """
         The object can either be initialized with individual params, or by
         passing the dict that is returned by swiftclient.
         """
-        self.client = client
+        self.manager = manager
-        self.container = container
         self.name = name
-        self.total_bytes = total_bytes
+        self.bytes = total_bytes or 0
         self.content_type = content_type
         self.last_modified = last_modified
-        self.etag = etag
+        self.hash = etag
         if attdict:
             self._read_attdict(attdict)
 
@@ -165,7 +202,8 @@ class FakeService(object):
 
 class FakeCSClient(FakeService):
     def __init__(self, *args, **kwargs):
-        super(FakeCSClient, self).__init__(*args, **kwargs)
+        ident = FakeIdentity()
+        super(FakeCSClient, self).__init__(ident, *args, **kwargs)
 
         def dummy(self):
             pass
@@ -202,21 +240,10 @@ class FakeBulkDeleter(BulkDeleter):
         self.completed = True
 
 
-class FakeEntryPoint(object):
-    def __init__(self, name):
-        self.name = name
-
-    def load(self):
-        def dummy(*args, **kwargs):
-            return self.name
-        return dummy
-
-fakeEntryPoints = [FakeEntryPoint("a"), FakeEntryPoint("b"),
-        FakeEntryPoint("c")]
-
-
 class FakeManager(object):
-    api = FakeClient()
+    def __init__(self, *args, **kwargs):
+        super(FakeManager, self).__init__(*args, **kwargs)
+        self.api = FakeClient()
 
     def list(self):
         pass
@@ -241,25 +268,6 @@ class FakeException(BaseException):
     pass
 
 
-class FakeServiceCatalog(object):
-    def __init__(self, *args, **kwargs):
-        pass
-
-    def get_token(self):
-        return "fake_token"
-
-    def url_for(self, attr=None, filter_value=None,
-            service_type=None, endpoint_type="publicURL",
-            service_name=None, volume_service_name=None):
-        if filter_value == "ALL":
-            raise exc.AmbiguousEndpoints
-        elif filter_value == "KEY":
-            raise KeyError
-        elif filter_value == "EP":
-            raise exc.EndpointNotFound
-        return "http://example.com"
-
-
 class FakeKeyring(object):
     password_set = False
 
@@ -315,7 +323,8 @@ class FakeDatabaseClient(CloudDatabaseClient):
     def __init__(self, *args, **kwargs):
         self._manager = FakeDatabaseManager(self)
         self._flavor_manager = FakeManager()
-        super(FakeDatabaseClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeDatabaseClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -326,8 +335,9 @@ class FakeNovaVolumeClient(BaseClient):
 
 class FakeBlockStorageManager(CloudBlockStorageManager):
     def __init__(self, api=None, *args, **kwargs):
+        ident = FakeIdentity()
         if api is None:
-            api = FakeBlockStorageClient()
+            api = FakeBlockStorageClient(ident)
         super(FakeBlockStorageManager, self).__init__(api, *args, **kwargs)
 
 
@@ -350,13 +360,15 @@ class FakeBlockStorageClient(CloudBlockStorageClient):
     def __init__(self, *args, **kwargs):
         self._types_manager = FakeManager()
         self._snapshot_manager = FakeManager()
-        super(FakeBlockStorageClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeBlockStorageClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
 class FakeLoadBalancerClient(CloudLoadBalancerClient):
     def __init__(self, *args, **kwargs):
-        super(FakeLoadBalancerClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeLoadBalancerClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -409,7 +421,8 @@ class FakeStatusChanger(object):
 
 class FakeDNSClient(CloudDNSClient):
     def __init__(self, *args, **kwargs):
-        super(FakeDNSClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeDNSClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -447,7 +460,8 @@ class FakeDNSDevice(FakeLoadBalancer):
 
 class FakeCloudNetworkClient(CloudNetworkClient):
     def __init__(self, *args, **kwargs):
-        super(FakeCloudNetworkClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeCloudNetworkClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -463,8 +477,9 @@ class FakeCloudNetwork(CloudNetwork):
 
 class FakeAutoScaleClient(AutoScaleClient):
     def __init__(self, *args, **kwargs):
+        ident = FakeIdentity()
         self._manager = FakeManager()
-        super(FakeAutoScaleClient, self).__init__(*args, **kwargs)
+        super(FakeAutoScaleClient, self).__init__(ident, *args, **kwargs)
 
 
 class FakeAutoScalePolicy(AutoScalePolicy):
@@ -500,7 +515,8 @@ class FakeScalingGroup(ScalingGroup):
 
 class FakeCloudMonitorClient(CloudMonitorClient):
     def __init__(self, *args, **kwargs):
-        super(FakeCloudMonitorClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeCloudMonitorClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -547,20 +563,10 @@ class FakeQueueClaim(QueueClaim):
                 **kwargs)
 
 
-class FakeQueueMessage(QueueMessage):
-    def __init__(self, *args, **kwargs):
-        id_ = utils.random_unicode()
-        href = "http://example.com/%s" % id_
-        info = kwargs.pop("info", {"href": href})
-        info["name"] = utils.random_unicode()
-        mgr = kwargs.pop("manager", FakeQueueManager())
-        super(FakeQueueMessage, self).__init__(manager=mgr, info=info, *args,
-                **kwargs)
-
-
 class FakeQueueClient(QueueClient):
     def __init__(self, *args, **kwargs):
-        super(FakeQueueClient, self).__init__("fakeuser",
+        ident = FakeIdentity()
+        super(FakeQueueClient, self).__init__(ident, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -584,8 +590,10 @@ class FakeImage(Image):
 
 
 class FakeImageClient(ImageClient):
-    def __init__(self, *args, **kwargs):
+    def __init__(self, identity=None, *args, **kwargs):
-        super(FakeImageClient, self).__init__("fakeuser",
+        if identity is None:
+            identity = FakeIdentity()
+        super(FakeImageClient, self).__init__(identity, "fakeuser",
                 "fakepassword", *args, **kwargs)
 
 
@@ -615,15 +623,43 @@ class FakeImageManager(ImageManager):
         self.id = utils.random_ascii()
 
 
-class FakeIdentity(RaxIdentity):
+class FakeIdentityService(Service):
+    def __init__(self, identity=None, *args, **kwargs):
+        self.identity = identity or FakeIdentity()
+        self.name = "fake"
+        self.prefix = ""
+        self.service_type = "fake"
+        self.clients = {}
+        self.endpoints = utils.DotDict()
+
+
+class FakeEndpoint(Endpoint):
+    def __init__(self, ep_dict=None, service=None, region=None, identity=None):
+        if ep_dict is None:
+            ep_dict = {}
+        if identity is None:
+            identity = FakeIdentity()
+        if service is None:
+            service = FakeIdentityService(identity)
+        if region is None:
+            region = "fake_region"
+        super(FakeEndpoint, self).__init__(ep_dict, service, region, identity)
+
+
+class FakeRaxIdentity(RaxIdentity):
+    pass
+
+
+class FakeIdentity(BaseIdentity):
     """Class that returns canned authentication responses."""
     def __init__(self, *args, **kwargs):
         super(FakeIdentity, self).__init__(*args, **kwargs)
         self._good_username = "fakeuser"
         self._good_password = "fakeapikey"
         self._default_region = random.choice(("DFW", "ORD"))
+        self.services = {"fake": FakeIdentityService(self)}
 
-    def authenticate(self):
+    def authenticate(self, connect=False):
         if ((self.username == self._good_username) and
                 (self.password == self._good_password)):
             self._parse_response(self.fake_response())
@@ -808,6 +844,9 @@ fake_identity_response = {u'access':
             'region': 'DFW',
             'tenantId': 'MossoCloudFS_abc'},
             {u'publicURL': 'https://cdn1.clouddrive.com/v1/MossoCloudFS_abc',
+            'region': 'FAKE',
+            'tenantId': 'MossoCloudFS_abc'},
+            {u'publicURL': 'https://cdn1.clouddrive.com/v1/MossoCloudFS_abc',
             'region': 'SYD',
             'tenantId': 'MossoCloudFS_abc'},
             {u'publicURL': 'https://cdn2.clouddrive.com/v1/MossoCloudFS_abc',

awx/lib/site-packages/pyrax/http.py

@@ -1,4 +1,4 @@
-# Copyright 2014 Rackspace
+# Copyright (c)2014 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -17,6 +17,7 @@
 Wrapper around the requests library. Used for making all HTTP calls.
 """
 
+import logging
 import json
 import requests
 
@@ -48,7 +49,7 @@ def request(method, uri, *args, **kwargs):
     req_method = req_methods[method.upper()]
     raise_exception = kwargs.pop("raise_exception", True)
     kwargs["headers"] = kwargs.get("headers", {})
-    http_log_req(args, kwargs)
+    http_log_req(method, uri, args, kwargs)
     data = None
     if "data" in kwargs:
         # The 'data' kwarg is used when you don't want json encoding.
@@ -72,27 +73,26 @@ def request(method, uri, *args, **kwargs):
     return resp, body
 
 
-def http_log_req(args, kwargs):
+def http_log_req(method, uri, args, kwargs):
     """
     When pyrax.get_http_debug() is True, outputs the equivalent `curl`
     command for the API request being made.
     """
     if not pyrax.get_http_debug():
         return
-    string_parts = ["curl -i"]
+    string_parts = ["curl -i -X %s" % method]
     for element in args:
-        if element in ("GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"):
+        string_parts.append("%s" % element)
-            string_parts.append(" -X %s" % element)
-        else:
-            string_parts.append(" %s" % element)
-
     for element in kwargs["headers"]:
-        header = " -H '%s: %s'" % (element, kwargs["headers"][element])
+        header = "-H '%s: %s'" % (element, kwargs["headers"][element])
         string_parts.append(header)
-
+    string_parts.append(uri)
-    pyrax._logger.debug("\nREQ: %s\n" % "".join(string_parts))
+    log = logging.getLogger("pyrax")
+    log.debug("\nREQ: %s\n" % " ".join(string_parts))
     if "body" in kwargs:
         pyrax._logger.debug("REQ BODY: %s\n" % (kwargs["body"]))
+    if "data" in kwargs:
+        pyrax._logger.debug("REQ DATA: %s\n" % (kwargs["data"]))
 
 
 def http_log_resp(resp, body):
@@ -102,4 +102,7 @@ def http_log_resp(resp, body):
     """
     if not pyrax.get_http_debug():
         return
-    pyrax._logger.debug("RESP: %s %s\n", resp, body)
+    log = logging.getLogger("pyrax")
+    log.debug("RESP: %s\n%s", resp, resp.headers)
+    if body:
+        log.debug("RESP BODY: %s", body)

awx/lib/site-packages/pyrax/identity/keystone_identity.py

@@ -1,12 +1,14 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
+from __future__ import absolute_import
+
 import pyrax
-from pyrax.base_identity import BaseAuth
+from ..base_identity import BaseIdentity
-import pyrax.exceptions as exc
+from .. import exceptions as exc
 
 
-class KeystoneIdentity(BaseAuth):
+class KeystoneIdentity(BaseIdentity):
     """
     Implements the Keystone-specific behaviors for Identity. In most
     cases you will want to create specific subclasses to implement the

awx/lib/site-packages/pyrax/identity/rax_identity.py

@@ -1,17 +1,21 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-from six.moves import configparser
+from __future__ import absolute_import
 
-from pyrax.base_identity import BaseAuth
+from six.moves import configparser as ConfigParser
-from pyrax.base_identity import User
+
-import pyrax.exceptions as exc
+import pyrax
-import pyrax.utils as utils
+from ..base_identity import BaseIdentity
+from ..base_identity import User
+from ..cloudnetworks import CloudNetworkClient
+from .. import exceptions as exc
+from .. import utils as utils
 
 AUTH_ENDPOINT = "https://identity.api.rackspacecloud.com/v2.0/"
 
 
-class RaxIdentity(BaseAuth):
+class RaxIdentity(BaseIdentity):
     """
     This class handles all of the authentication requirements for working
     with the Rackspace Cloud.
@@ -21,7 +25,8 @@ class RaxIdentity(BaseAuth):
 
 
     def _get_auth_endpoint(self):
-        return self._auth_endpoint or AUTH_ENDPOINT
+        return (self._auth_endpoint or pyrax.get_setting("auth_endpoint")
+                or AUTH_ENDPOINT)
 
 
     def _read_credential_file(self, cfg):
@@ -31,12 +36,12 @@ class RaxIdentity(BaseAuth):
         self.username = cfg.get("rackspace_cloud", "username")
         try:
             self.password = cfg.get("rackspace_cloud", "api_key", raw=True)
-        except configparser.NoOptionError as e:
+        except ConfigParser.NoOptionError as e:
             # Allow either the use of either 'api_key' or 'password'.
             self.password = cfg.get("rackspace_cloud", "password", raw=True)
 
 
-    def _get_credentials(self):
+    def _format_credentials(self):
         """
         Returns the current credentials in the format expected by the
         authentication service. Note that by default Rackspace credentials
@@ -47,23 +52,40 @@ class RaxIdentity(BaseAuth):
         if self._creds_style == "apikey":
             return {"auth": {"RAX-KSKEY:apiKeyCredentials":
                     {"username": "%s" % self.username,
-                    "apiKey": "%s" % self.password}}}
+                    "apiKey": "%s" % self.api_key}}}
         else:
             # Return in the default password-style
-            return super(RaxIdentity, self)._get_credentials()
+            return super(RaxIdentity, self)._format_credentials()
 
 
-    def authenticate(self):
+    def set_credentials(self, username, password=None, region=None,
+            tenant_id=None, authenticate=False):
+        """
+        Sets the username and password directly. Because Rackspace auth uses
+        the api_key, make sure that any old values are cleared.
+        """
+        self.api_key = None
+        super(RaxIdentity, self).set_credentials(username, password=password,
+                region=region, tenant_id=tenant_id, authenticate=authenticate)
+
+
+    def authenticate(self, username=None, password=None, api_key=None,
+            tenant_id=None, connect=False):
         """
         If the user's credentials include an API key, the default behavior will
         work. But if they are using a password, the initial attempt will fail,
         so try again, but this time using the standard password format.
+
+        The 'connect' parameter is retained for backwards compatibility. It no
+        longer has any effect.
         """
         try:
-            super(RaxIdentity, self).authenticate()
+            super(RaxIdentity, self).authenticate(username=username,
+                    password=password, api_key=api_key, tenant_id=tenant_id)
         except exc.AuthenticationFailed:
             self._creds_style = "password"
-            super(RaxIdentity, self).authenticate()
+            super(RaxIdentity, self).authenticate(username=username,
+                    password=password, api_key=api_key, tenant_id=tenant_id)
 
 
     def auth_with_token(self, token, tenant_id=None, tenant_name=None):
@@ -80,16 +102,16 @@ class RaxIdentity(BaseAuth):
         # object_store endpoints. We can then add these to the initial
         # endpoints returned by the primary tenant ID, and then continue with
         # the auth process.
-        main_resp = self._call_token_auth(token, tenant_id, tenant_name)
+        main_resp, main_body = self._call_token_auth(token, tenant_id,
-        main_body = main_resp.json()
+                tenant_name)
         # Get the swift tenant ID
         roles = main_body["access"]["user"]["roles"]
         ostore = [role for role in roles
                 if role["name"] == "object-store:default"]
         if ostore:
             ostore_tenant_id = ostore[0]["tenantId"]
-            ostore_resp = self._call_token_auth(token, ostore_tenant_id, None)
+            ostore_resp, ostore_body = self._call_token_auth(token,
-            ostore_body = ostore_resp.json()
+                    ostore_tenant_id, None)
             ostore_cat = ostore_body["access"]["serviceCatalog"]
             main_cat = main_body["access"]["serviceCatalog"]
             main_cat.extend(ostore_cat)
@@ -106,13 +128,44 @@ class RaxIdentity(BaseAuth):
             self._default_region = defreg
 
 
+    def get_client(self, service, region, public=True, cached=True):
+        """
+        Returns the client object for the specified service and region.
+
+        By default the public endpoint is used. If you wish to work with a
+        services internal endpoints, specify `public=False`.
+
+        By default, if a client has already been created for the given service,
+        region, and public values, that will be returned. To force a new client
+        to be created, pass 'cached=False'.
+        """
+        client_class = None
+        # Cloud Networks currently uses nova-networks, so it doesn't appear as
+        # a separate entry in the service catalog. This hack will allow context
+        # objects to continue to work with Rackspace Cloud Networks. When the
+        # Neutron service is implemented, this hack will have to be removed.
+        if service in ("compute:networks", "networks", "network",
+                "cloudnetworks", "cloud_networks"):
+            service = "compute"
+            client_class = CloudNetworkClient
+        return super(RaxIdentity, self).get_client(service, region,
+                public=public, cached=cached, client_class=client_class)
+
+
     def find_user_by_name(self, name):
         """
         Returns a User object by searching for the supplied user name. Returns
         None if there is no match for the given name.
         """
-        uri = "users?name=%s" % name
+        return self.get_user(username=name)
-        return self._find_user(uri)
+
+
+    def find_user_by_email(self, email):
+        """
+        Returns a User object by searching for the supplied user's email
+        address. Returns None if there is no match for the given ID.
+        """
+        return self.get_user(email=email)
 
 
     def find_user_by_id(self, uid):
@@ -120,18 +173,42 @@ class RaxIdentity(BaseAuth):
         Returns a User object by searching for the supplied user ID. Returns
         None if there is no match for the given ID.
         """
-        uri = "users/%s" % uid
+        return self.get_user(user_id=uid)
-        return self._find_user(uri)
 
 
-    def _find_user(self, uri):
+    def get_user(self, user_id=None, username=None, email=None):
-        """Handles the 'find' code for both name and ID searches."""
+        """
-        resp = self.method_get(uri)
+        Returns the user specified by either ID, username or email.
-        if resp.status_code in (403, 404):
+
-            return None
+        Since more than user can have the same email address, searching by that
-        jusers = resp.json()
+        term will return a list of 1 or more User objects. Searching by
-        user_info = jusers["user"]
+        username or ID will return a single User.
-        return User(self, user_info)
+
+        If a user_id that doesn't belong to the current account is searched
+        for, a Forbidden exception is raised. When searching by username or
+        email, a NotFound exception is raised if there is no matching user.
+        """
+        if user_id:
+            uri = "/users/%s" % user_id
+        elif username:
+            uri = "/users?name=%s" % username
+        elif email:
+            uri = "/users?email=%s" % email
+        else:
+            raise ValueError("You must include one of 'user_id', "
+                    "'username', or 'email' when calling get_user().")
+        resp, resp_body = self.method_get(uri)
+        if resp.status_code == 404:
+            raise exc.NotFound("No such user exists.")
+        users = resp_body.get("users", [])
+        if users:
+            return [User(self, user) for user in users]
+        else:
+            user = resp_body.get("user", {})
+            if user:
+                return User(self, user)
+            else:
+                raise exc.NotFound("No such user exists.")
 
 
     def update_user(self, user, email=None, username=None,
@@ -151,24 +228,26 @@ class RaxIdentity(BaseAuth):
         if enabled is not None:
             upd["enabled"] = enabled
         data = {"user": upd}
-        resp = self.method_put(uri, data=data)
+        resp, resp_body = self.method_put(uri, data=data)
-        return User(self, resp.json())
+        if resp.status_code in (401, 403, 404):
+            raise exc.AuthorizationFailure("You are not authorized to update "
+                    "users.")
+        return User(self, resp_body)
 
 
-    def list_credentials(self, user):
+    def reset_api_key(self, user=None):
         """
-        Returns a user's non-password credentials.
+        Resets the API key for the specified user, or if no user is specified,
-        """
+        for the current user. Returns the newly-created API key.
-        user_id = utils.get_id(user)
-        uri = "users/%s/OS-KSADM/credentials" % user_id
-        return self.method_get(uri)
 
-
+        Resetting an API key does not invalidate any authenticated sessions,
-    def get_user_credentials(self, user):
+        nor does it revoke any tokens.
         """
-        Returns a user's non-password credentials.
+        if user is None:
-        """
+            user_id = utils.get_id(self)
-        user_id = utils.get_id(user)
+        else:
-        base_uri = "users/%s/OS-KSADM/credentials/RAX-KSKEY:apiKeyCredentials"
+            user_id = utils.get_id(user)
-        uri = base_uri % user_id
+        uri = "users/%s/OS-KSADM/credentials/" % user_id
-        return self.method_get(uri)
+        uri += "RAX-KSKEY:apiKeyCredentials/RAX-AUTH/reset"
+        resp, resp_body = self.method_post(uri)
+        return resp_body.get("RAX-KSKEY:apiKeyCredentials", {}).get("apiKey")

awx/lib/site-packages/pyrax/image.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2014 Rackspace
+# Copyright (c)2014 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -20,6 +20,7 @@
 from functools import wraps
 
 import pyrax
+from pyrax.object_storage import StorageObject
 from pyrax.client import BaseClient
 import pyrax.exceptions as exc
 from pyrax.manager import BaseManager
@@ -251,6 +252,45 @@ class ImageManager(BaseManager):
         return ret
 
 
+    def create(self, name, img_format=None, img_container_format=None,
+            data=None, container=None, obj=None, metadata=None):
+        """
+        Creates a new image with the specified name. The image data can either
+        be supplied directly in the 'data' parameter, or it can be an image
+        stored in the object storage service. In the case of the latter, you
+        can either supply the container and object names, or simply a
+        StorageObject reference.
+
+        You may specify the image and image container formats; if unspecified,
+        the default of "vhd" for image format and "bare" for image container
+        format will be used.
+
+        NOTE: This is blocking, and may take a while to complete.
+        """
+        if img_format is None:
+            img_format = "vhd"
+        if img_container_format is None:
+            img_container_format = "bare"
+        headers = {
+                "X-Image-Meta-name": name,
+                "X-Image-Meta-disk_format": img_format,
+                "X-Image-Meta-container_format": img_container_format,
+                }
+        if data:
+            img_data = data
+        else:
+            ident = self.api.identity
+            region = self.api.region_name
+            clt = ident.get_client("object_store", region)
+            if not isinstance(obj, StorageObject):
+                obj = clt.get_object(container, obj)
+            img_data = obj.fetch()
+        uri = "%s/images" % self.uri_base
+        resp, resp_body = self.api.method_post(uri, headers=headers,
+                data=img_data)
+
+
+
     def update(self, img, value_dict):
         """
         Accepts an image reference (object or ID) and  dictionary of key/value
@@ -294,7 +334,8 @@ class ImageManager(BaseManager):
             raise exc.InvalidImageMemberStatus("The status value must be one "
                     "of 'accepted', 'rejected', or 'pending'. Received: '%s'" %
                     status)
-        project_id = pyrax.identity.tenant_id
+        api = self.api
+        project_id = api.identity.tenant_id
         uri = "/%s/%s/members/%s" % (self.uri_base, img_id, project_id)
         body = {"status": status}
         try:
@@ -385,7 +426,10 @@ class ImageTasksManager(BaseManager):
         if cont:
             # Verify that it exists. If it doesn't, a NoSuchContainer exception
             # will be raised.
-            pyrax.cloudfiles.get_container(cont)
+            api = self.api
+            rgn = api.region_name
+            cf = api.identity.object_store[rgn].client
+            cf.get_container(cont)
         return super(ImageTasksManager, self).create(name, *args, **kwargs)
 
 
@@ -518,6 +562,19 @@ class ImageClient(BaseClient):
         return self._manager.update(img, value_dict)
 
 
+    def create(self, name, img_format=None, data=None, container=None,
+            obj=None, metadata=None):
+        """
+        Creates a new image with the specified name. The image data can either
+        be supplied directly in the 'data' parameter, or it can be an image
+        stored in the object storage service. In the case of the latter, you
+        can either supply the container and object names, or simply a
+        StorageObject reference.
+        """
+        return self._manager.create(name, img_format, data=data,
+                container=container, obj=obj)
+
+
     def change_image_name(self, img, newname):
         """
         Image name can be changed via the update() method. This is simply a

awx/lib/site-packages/pyrax/manager.py

@@ -1,7 +1,7 @@
 # Copyright 2010 Jacob Kaplan-Moss
 
 # Copyright 2011 OpenStack LLC.
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #

awx/lib/site-packages/pyrax/queueing.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -21,7 +21,7 @@ from functools import wraps
 import json
 import os
 import re
-import urlparse
+from six.moves import urllib_parse as urlparse
 
 import pyrax
 from pyrax.client import BaseClient

awx/lib/site-packages/pyrax/resource.py

@@ -1,7 +1,7 @@
 # Copyright 2010 Jacob Kaplan-Moss
 
 # Copyright 2011 OpenStack LLC.
-# Copyright 2012 Rackspace
+# Copyright (c)2012 Rackspace US, Inc.
 
 # All Rights Reserved.
 #
@@ -67,9 +67,11 @@ class BaseResource(object):
         Takes the dict returned by the API call and sets the
         corresponding attributes on the object.
         """
-        for (key, val) in info.iteritems():
+        for (key, val) in six.iteritems(info):
             if isinstance(key, six.text_type):
                 key = key.encode(pyrax.get_encoding())
+            elif isinstance(key, bytes):
+                key = key.decode("utf-8")
             setattr(self, key, val)
 
 

awx/lib/site-packages/pyrax/utils.py

@@ -111,6 +111,111 @@ class SelfDeletingTempDirectory(object):
         shutil.rmtree(self.name)
 
 
+
+class DotDict(dict):
+    """
+    Dictionary subclass that allows accessing keys via dot notation.
+
+    If the key is not present, an AttributeError is raised.
+    """
+    _att_mapper = {}
+    _fail = object()
+
+    def __init__(self, *args, **kwargs):
+        super(DotDict, self).__init__(*args, **kwargs)
+
+    def __getattr__(self, att):
+        att = self._att_mapper.get(att, att)
+        ret = self.get(att, self._fail)
+        if ret is self._fail:
+            raise AttributeError("'%s' object has no attribute '%s'" %
+                    (self.__class__.__name__, att))
+        return ret
+
+    __setattr__ = dict.__setitem__
+    __delattr__ = dict.__delitem__
+
+
+
+class ResultsIterator(object):
+    """
+    This object will iterate over all the results for a given type of listing,
+    no matter how many items exist.
+
+    This is an abstract class; subclasses must define the _init_methods()
+    method to specify what manager method should be called to get the next
+    batch of results. Both the equivalent of 'list()' and '_list()' may be
+    specified You may also specify any extra args to be sent when this method
+    is called.
+
+    By default the marker will be unspecified, and the limit will be 1000. You
+    can override either by specifying them during instantiation.
+
+    The 'kwargs' will be converted to attributes. E.g., in this call:
+        rit = ResultsIterator(mgr, foo="bar")
+    will result in the object having a 'foo' attribute with the value of 'bar'.
+    """
+    def __init__(self, manager, marker=None, limit=1000, **kwargs):
+        self.manager = manager
+        self.marker = marker
+        self.limit = limit
+        for att, val in list(kwargs.items()):
+            setattr(self, att, val)
+        self.results = []
+        self.list_method = None
+        self._list_method = None
+        self.marker_att = "id"
+        self.extra_args = tuple()
+        self._init_methods()
+        self.next_uri = ""
+
+
+    def _init_methods(self):
+        """
+        Must be implemented in subclasses. For results that return a URI for
+        the next batch of results, the lower-level '_list_method' will be
+        called, using that URI. Otherwise, the 'list_method' will be called,
+        with the paging info from the prior call.
+
+        If your class uses an attribute other than 'id' as the marker, set this
+        object's 'marker_att' to that attribute.
+        """
+        raise NotImplementedError()
+
+
+    def __iter__(self):
+        return self
+
+
+    def next(self):
+        """
+        Return the next available item. If there are no more items in the
+        local 'results' list, check if there is a 'next_uri' value. If so,
+        use that to get the next page of results from the API, and return
+        the first item from that query.
+        """
+        try:
+            return self.results.pop(0)
+        except IndexError:
+            if self.next_uri is None:
+                raise StopIteration()
+            else:
+                if not self.next_uri:
+                    self.results = self.list_method(marker=self.marker,
+                            limit=self.limit, prefix=self.prefix)
+                else:
+                    args = self.extra_args
+                    self.results = self._list_method(self.next_uri, *args)
+                if self.results:
+                    last_res = self.results[-1]
+                    self.marker = getattr(last_res, self.marker_att)
+        # We should have more results.
+        try:
+            return self.results.pop(0)
+        except IndexError:
+            raise StopIteration()
+
+
 def get_checksum(content, encoding="utf8", block_size=8192):
     """
     Returns the MD5 checksum in hex for the given content. If 'content'
@@ -161,7 +266,7 @@ def _join_chars(chars, length):
     """
     Used by the random character functions.
     """
-    mult = (length / len(chars)) + 1
+    mult = int(length / len(chars)) + 1
     mult_chars = chars * mult
     return "".join(random.sample(mult_chars, length))
 
@@ -174,7 +279,7 @@ def random_unicode(length=20):
     up to code point 1000.
     """
     def get_char():
-        return unichr(random.randint(32, 1000))
+        return six.unichr(random.randint(32, 1000))
     chars = u"".join([get_char() for ii in six.moves.range(length)])
     return _join_chars(chars, length)
 

awx/lib/site-packages/pyrax/version.py

@@ -1,4 +1,4 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-version = "1.7.2"
+version = "1.9.0"