External-Mirrors/awx
2
0
Fork 0
mirror of https://github.com/ansible/awx.git synced 2026-01-17 12:41:19 -03:30
Code Issues Packages Projects Releases Wiki Activity

Upgrade pyrax to 1.9.0

Browse Source
This commit is contained in:
Matthew Jones 2014-08-06 15:35:07 -04:00
parent e1da8d169f
commit 7cf1df23ab
26 changed files with 4915 additions and 543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
awx/lib/site-packages/README
View File

@ -43,7 +43,7 @@ pexpect==3.3 (pexpect/*, excluded pxssh.py, fdpexpect.py, FSM.py, screen.py,
ANSI.py)
pip==1.5.4 (pip/*, excluded bin/pip*)
prettytable==0.7.2 (prettytable.py)
pyrax==1.7.2 (pyrax/*)
pyrax==1.9.0 (pyrax/*)
python-dateutil==2.2 (dateutil/*)
python-novaclient==2.17.0 (novaclient/*, excluded bin/nova)
python-swiftclient==2.0.3 (swiftclient/*, excluded bin/swift)

277
awx/lib/site-packages/pyrax/__init__.py
View File

@ -1,6 +1,6 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -27,22 +27,17 @@ built on the Rackspace / OpenStack Cloud.<br />
The source code for <b>pyrax</b> can be found at:
http://github.com/rackspace/pyrax
\package cf_wrapper
This module wraps <b>swiftclient</b>, the Python client for OpenStack / Swift,
providing an object-oriented interface to the Swift object store.
It also adds in CDN functionality that is Rackspace-specific.
"""
from __future__ import absolute_import
from functools import wraps
import inspect
import logging
import os
import re
import six.moves.configparser as ConfigParser
import warnings
from six.moves import configparser
# keyring is an optional import
try:
import keyring
@ -59,21 +54,22 @@ try:
from . import http
from . import version
import cf_wrapper.client as _cf
from novaclient import exceptions as _cs_exceptions
from novaclient import auth_plugin as _cs_auth_plugin
from novaclient.shell import OpenStackComputeShell as _cs_shell
from novaclient.v1_1 import client as _cs_client
from novaclient.v1_1.servers import Server as CloudServer
from autoscale import AutoScaleClient
from clouddatabases import CloudDatabaseClient
from cloudloadbalancers import CloudLoadBalancerClient
from cloudblockstorage import CloudBlockStorageClient
from clouddns import CloudDNSClient
from cloudnetworks import CloudNetworkClient
from cloudmonitoring import CloudMonitorClient
from image import ImageClient
from queueing import QueueClient
from .autoscale import AutoScaleClient
from .clouddatabases import CloudDatabaseClient
from .cloudloadbalancers import CloudLoadBalancerClient
from .cloudblockstorage import CloudBlockStorageClient
from .clouddns import CloudDNSClient
from .cloudnetworks import CloudNetworkClient
from .cloudmonitoring import CloudMonitorClient
from .image import ImageClient
from .object_storage import StorageClient
from .queueing import QueueClient
except ImportError:
# See if this is the result of the importing of version.py in setup.py
callstack = inspect.stack()
@ -118,6 +114,8 @@ regions = tuple()
services = tuple()
_client_classes = {
"compute": _cs_client.Client,
"object_store": StorageClient,
"database": CloudDatabaseClient,
"load_balancer": CloudLoadBalancerClient,
"volume": CloudBlockStorageClient,
@ -168,7 +166,7 @@ class Settings(object):
"verify_ssl": "CLOUD_VERIFY_SSL",
"use_servicenet": "USE_SERVICENET",
}
_settings = {"default": dict.fromkeys(env_dct.keys())}
_settings = {"default": dict.fromkeys(list(env_dct.keys()))}
_default_set = False
@ -181,8 +179,10 @@ class Settings(object):
if env is None:
env = self.environment
try:
return self._settings[env][key]
ret = self._settings[env][key]
except KeyError:
ret = None
if ret is None:
# See if it's set in the environment
if key == "identity_class":
# This is defined via the identity_type
@ -193,9 +193,10 @@ class Settings(object):
else:
env_var = self.env_dct.get(key)
try:
return os.environ[env_var]
ret = os.environ[env_var]
except KeyError:
return None
ret = None
return ret
def set(self, key, val, env=None):
@ -210,7 +211,7 @@ class Settings(object):
else:
if env not in self._settings:
raise exc.EnvironmentNotFound("There is no environment named "
"'%s'." % env)
"'%s'." % env)
dct = self._settings[env]
if key not in dct:
raise exc.InvalidSetting("The setting '%s' is not defined." % key)
@ -257,7 +258,7 @@ class Settings(object):
@property
def environments(self):
return self._settings.keys()
return list(self._settings.keys())
def read_config(self, config_file):
@ -265,17 +266,17 @@ class Settings(object):
Parses the specified configuration file and stores the values. Raises
an InvalidConfigurationFile exception if the file is not well-formed.
"""
cfg = configparser.SafeConfigParser()
cfg = ConfigParser.SafeConfigParser()
try:
cfg.read(config_file)
except configparser.MissingSectionHeaderError as e:
except ConfigParser.MissingSectionHeaderError as e:
# The file exists, but doesn't have the correct format.
raise exc.InvalidConfigurationFile(e)
def safe_get(section, option, default=None):
try:
return cfg.get(section, option)
except (configparser.NoSectionError, configparser.NoOptionError):
except (ConfigParser.NoSectionError, ConfigParser.NoOptionError):
return default
# A common mistake is including credentials in the config file. If any
@ -297,8 +298,9 @@ class Settings(object):
dct = self._settings[section_name] = {}
dct["region"] = safe_get(section, "region", default_region)
ityp = safe_get(section, "identity_type")
dct["identity_type"] = _id_type(ityp)
dct["identity_class"] = _import_identity(ityp)
if ityp:
dct["identity_type"] = _id_type(ityp)
dct["identity_class"] = _import_identity(ityp)
# Handle both the old and new names for this setting.
debug = safe_get(section, "debug")
if debug is None:
@ -377,18 +379,46 @@ def set_default_region(region):
default_region = region
def _create_identity():
def create_context(id_type=None, env=None, username=None, password=None,
tenant_id=None, tenant_name=None, api_key=None, verify_ssl=None):
"""
Returns an instance of the specified identity class, or if none is
specified, an instance of the current setting for 'identity_class'.
You may optionally set the environment by passing the name of that
environment in the 'env' parameter.
"""
if env:
set_environment(env)
return _create_identity(id_type=id_type, username=username,
password=password, tenant_id=tenant_id, tenant_name=tenant_name,
api_key=api_key, verify_ssl=verify_ssl, return_context=True)
def _create_identity(id_type=None, username=None, password=None, tenant_id=None,
tenant_name=None, api_key=None, verify_ssl=None,
return_context=False):
"""
Creates an instance of the current identity_class and assigns it to the
module-level name 'identity'.
module-level name 'identity' by default. If 'return_context' is True, the
module-level 'identity' is untouched, and instead the instance is returned.
"""
global identity
cls = settings.get("identity_class")
if id_type:
cls = _import_identity(id_type)
else:
cls = settings.get("identity_class")
if not cls:
raise exc.IdentityClassNotDefined("No identity class has "
"been defined for the current environment.")
verify_ssl = get_setting("verify_ssl")
identity = cls(verify_ssl=verify_ssl)
if verify_ssl is None:
verify_ssl = get_setting("verify_ssl")
context = cls(username=username, password=password, tenant_id=tenant_id,
tenant_name=tenant_name, api_key=api_key, verify_ssl=verify_ssl)
if return_context:
return context
else:
global identity
identity = context
def _assure_identity(fnc):
@ -412,13 +442,16 @@ def _require_auth(fnc):
return _wrapped
@_assure_identity
def _safe_region(region=None):
def _safe_region(region=None, context=None):
"""Value to use when no region is specified."""
ret = region or settings.get("region")
context = context or identity
if not ret:
# Nothing specified; get the default from the identity object.
ret = identity.get_default_region()
if not context:
_create_identity()
context = identity
ret = context.get_default_region()
if not ret:
# Use the first available region
try:
@ -434,8 +467,11 @@ def auth_with_token(token, tenant_id=None, tenant_name=None, region=None):
If you already have a valid token and either a tenant ID or name, you can
call this to configure the identity and available services.
"""
global regions, services
identity.auth_with_token(token, tenant_id=tenant_id,
tenant_name=tenant_name)
regions = tuple(identity.regions)
services = tuple(identity.services.keys())
connect_to_services(region=region)
@ -448,13 +484,15 @@ def set_credentials(username, api_key=None, password=None, region=None,
If the region is passed, it will authenticate against the proper endpoint
for that region, and set the default region for connections.
"""
global regions, services
pw_key = password or api_key
region = _safe_region(region)
tenant_id = tenant_id or settings.get("tenant_id")
identity.set_credentials(username=username, password=pw_key,
tenant_id=tenant_id, region=region)
if authenticate:
_auth_and_connect(region=region)
tenant_id=tenant_id, region=region, authenticate=authenticate)
regions = tuple(identity.regions)
services = tuple(identity.services.keys())
connect_to_services(region=region)
@_assure_identity
@ -478,10 +516,13 @@ def set_credential_file(cred_file, region=None, authenticate=True):
If the region is passed, it will authenticate against the proper endpoint
for that region, and set the default region for connections.
"""
global regions, services
region = _safe_region(region)
identity.set_credential_file(cred_file, region=region)
if authenticate:
_auth_and_connect(region=region)
identity.set_credential_file(cred_file, region=region,
authenticate=authenticate)
regions = tuple(identity.regions)
services = tuple(identity.services.keys())
connect_to_services(region=region)
def keyring_auth(username=None, region=None, authenticate=True):
@ -514,23 +555,6 @@ def keyring_auth(username=None, region=None, authenticate=True):
authenticate=authenticate)
def _auth_and_connect(region=None, connect=True):
"""
Handles the call to authenticate, and if successful, connects to the
various services.
"""
global default_region
identity.authenticated = False
default_region = region or default_region
try:
identity.authenticate()
except exc.AuthenticationFailed:
clear_credentials()
raise
if connect:
connect_to_services(region=region)
@_assure_identity
def authenticate(connect=True):
"""
@ -545,19 +569,11 @@ def authenticate(connect=True):
Normally after successful authentication, connections to the various
services will be made. However, passing False to the `connect` parameter
will skip the service connection step.
"""
_auth_and_connect(connect=connect)
def plug_hole_in_swiftclient_auth(clt, url):
The 'connect' parameter is retained for backwards compatibility. It no
longer has any effect.
"""
This is necessary because swiftclient has an issue when a token expires and
it needs to re-authenticate against Rackspace auth. It is a temporary
workaround until we can fix swiftclient.
"""
conn = clt.connection
conn.token = identity.token
conn.url = url
identity.authenticate()
def clear_credentials():
@ -610,46 +626,52 @@ def connect_to_services(region=None):
queues = connect_to_queues(region=region)
def _get_service_endpoint(svc, region=None, public=True):
def _get_service_endpoint(context, svc, region=None, public=True):
"""
Parses the services dict to get the proper endpoint for the given service.
"""
region = _safe_region(region)
url_type = {True: "public_url", False: "internal_url"}[public]
ep = identity.services.get(svc, {}).get("endpoints", {}).get(
region, {}).get(url_type)
# If a specific context is passed, use that. Otherwise, use the global
# identity reference.
context = context or identity
url_type = {True: "public", False: "private"}[public]
svc_obj = context.services.get(svc)
if not svc_obj:
return None
ep = svc_obj.endpoints.get(region, {}).get(url_type)
if not ep:
# Try the "ALL" region, and substitute the actual region
ep = identity.services.get(svc, {}).get("endpoints", {}).get(
"ALL", {}).get(url_type)
ep = svc_obj.endpoints.get("ALL", {}).get(url_type)
return ep
@_require_auth
def connect_to_cloudservers(region=None, **kwargs):
def connect_to_cloudservers(region=None, context=None, **kwargs):
"""Creates a client for working with cloud servers."""
context = context or identity
_cs_auth_plugin.discover_auth_systems()
id_type = get_setting("identity_type")
if id_type != "keystone":
auth_plugin = _cs_auth_plugin.load_plugin(id_type)
else:
auth_plugin = None
region = _safe_region(region)
mgt_url = _get_service_endpoint("compute", region)
region = _safe_region(region, context=context)
mgt_url = _get_service_endpoint(context, "compute", region)
cloudservers = None
if not mgt_url:
# Service is not available
return
insecure = not get_setting("verify_ssl")
cloudservers = _cs_client.Client(identity.username, identity.password,
project_id=identity.tenant_id, auth_url=identity.auth_endpoint,
cs_shell = _cs_shell()
extensions = cs_shell._discover_extensions("1.1")
cloudservers = _cs_client.Client(context.username, context.password,
project_id=context.tenant_id, auth_url=context.auth_endpoint,
auth_system=id_type, region_name=region, service_type="compute",
auth_plugin=auth_plugin, insecure=insecure,
auth_plugin=auth_plugin, insecure=insecure, extensions=extensions,
http_log_debug=_http_debug, **kwargs)
agt = cloudservers.client.USER_AGENT
cloudservers.client.USER_AGENT = _make_agent_name(agt)
cloudservers.client.management_url = mgt_url
cloudservers.client.auth_token = identity.token
cloudservers.client.auth_token = context.token
cloudservers.exceptions = _cs_exceptions
# Add some convenience methods
cloudservers.list_images = cloudservers.images.list
@ -672,54 +694,50 @@ def connect_to_cloudservers(region=None, **kwargs):
return [image for image in cloudservers.images.list()
if hasattr(image, "server")]
def find_images_by_name(expr):
"""
Returns a list of images whose name contains the specified expression.
The value passed is treated as a regular expression, allowing for more
specific searches than simple wildcards. The matching is done in a
case-insensitive manner.
"""
return [image for image in cloudservers.images.list()
if re.search(expr, image.name, re.I)]
cloudservers.list_base_images = list_base_images
cloudservers.list_snapshots = list_snapshots
cloudservers.find_images_by_name = find_images_by_name
cloudservers.identity = identity
return cloudservers
@_require_auth
def connect_to_cloudfiles(region=None, public=None):
"""
Creates a client for working with cloud files. The default is to connect
to the public URL; if you need to work with the ServiceNet connection, pass
False to the 'public' parameter or set the "use_servicenet" setting to True.
"""
"""Creates a client for working with CloudFiles/Swift."""
if public is None:
is_public = not bool(get_setting("use_servicenet"))
else:
is_public = public
region = _safe_region(region)
cf_url = _get_service_endpoint("object_store", region, public=is_public)
cloudfiles = None
if not cf_url:
# Service is not available
return
cdn_url = _get_service_endpoint("object_cdn", region)
ep_type = {True: "publicURL", False: "internalURL"}[is_public]
opts = {"tenant_id": identity.tenant_name, "auth_token": identity.token,
"endpoint_type": ep_type, "tenant_name": identity.tenant_name,
"object_storage_url": cf_url, "object_cdn_url": cdn_url,
"region_name": region}
verify_ssl = get_setting("verify_ssl")
cloudfiles = _cf.CFClient(identity.auth_endpoint, identity.username,
identity.password, tenant_name=identity.tenant_name,
preauthurl=cf_url, preauthtoken=identity.token, auth_version="2",
os_options=opts, verify_ssl=verify_ssl, http_log_debug=_http_debug)
cloudfiles.user_agent = _make_agent_name(cloudfiles.user_agent)
return cloudfiles
ret = _create_client(ep_name="object_store", region=region,
public=is_public)
if ret:
# Add CDN endpoints, if available
region = _safe_region(region)
ret.cdn_management_url = _get_service_endpoint(None, "object_cdn",
region, public=is_public)
return ret
@_require_auth
def _create_client(ep_name, region, public=True):
region = _safe_region(region)
ep = _get_service_endpoint(ep_name.split(":")[0], region, public=public)
ep = _get_service_endpoint(None, ep_name.split(":")[0], region,
public=public)
if not ep:
return
verify_ssl = get_setting("verify_ssl")
cls = _client_classes[ep_name]
client = cls(region_name=region, management_url=ep, verify_ssl=verify_ssl,
http_log_debug=_http_debug)
client = cls(identity, region_name=region, management_url=ep,
verify_ssl=verify_ssl, http_log_debug=_http_debug)
client.user_agent = _make_agent_name(client.user_agent)
return client
@ -769,34 +787,29 @@ def connect_to_queues(region=None, public=True):
return _create_client(ep_name="queues", region=region, public=public)
def client_class_for_service(service):
"""
Returns the client class registered for the given service, or None if there
is no such service, or if no class has been registered.
"""
return _client_classes.get(service)
def get_http_debug():
return _http_debug
@_assure_identity
def set_http_debug(val):
global _http_debug
_http_debug = val
# Set debug on the various services
identity.http_log_debug = val
if identity:
identity.http_log_debug = val
for svc in (cloudservers, cloudfiles, cloud_loadbalancers,
cloud_blockstorage, cloud_databases, cloud_dns, cloud_networks,
autoscale, images, queues):
if svc is not None:
svc.http_log_debug = val
# Need to manually add/remove the debug handler for swiftclient
swift_logger = _cf._swift_client.logger
if val:
for handler in swift_logger.handlers:
if isinstance(handler, logging.StreamHandler):
# Already present
return
swift_logger.addHandler(logging.StreamHandler())
swift_logger.setLevel(logging.DEBUG)
else:
for handler in swift_logger.handlers:
if isinstance(handler, logging.StreamHandler):
swift_logger.removeHandler(handler)
def get_encoding():

17
awx/lib/site-packages/pyrax/autoscale.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Rackspace
# Copyright (c)2013 Rackspace US, Inc.
# All Rights Reserved.
#
@ -17,6 +17,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import base64
import pyrax
from pyrax.client import BaseClient
from pyrax.cloudloadbalancers import CloudLoadBalancer
@ -441,9 +443,9 @@ class ScalingGroupManager(BaseManager):
largs = scaling_group.launchConfiguration.get("args", {})
srv_args = largs.get("server", {})
lb_args = largs.get("loadBalancers", {})
flav = "%s" % flavor or srv_args.get("flavorRef")
dconf = disk_config or srv_args.get("OS-DCF:diskConfig")
pers = personality or srv_args.get("personality")
flav = flavor or srv_args.get("flavorRef")
dconf = disk_config or srv_args.get("OS-DCF:diskConfig", "AUTO")
pers = personality or srv_args.get("personality", [])
body = {"type": "launch_server",
"args": {
"server": {
@ -451,13 +453,14 @@ class ScalingGroupManager(BaseManager):
"imageRef": image or srv_args.get("imageRef"),
"flavorRef": flav,
"OS-DCF:diskConfig": dconf,
"personality": pers,
"networks": networks or srv_args.get("networks"),
"metadata": metadata or srv_args.get("metadata"),
},
"loadBalancers": load_balancers or lb_args,
},
}
if pers:
body["args"]["server"]["personality"] = pers
key_name = key_name or srv_args.get("key_name")
if key_name:
body["args"]["server"] = key_name
@ -765,6 +768,10 @@ class ScalingGroupManager(BaseManager):
metadata = {}
if personality is None:
personality = []
else:
for file in personality:
if "contents" in file:
file["contents"] = base64.b64encode(file["contents"])
if scaling_policies is None:
scaling_policies = []
group_config = self._create_group_config_body(name, cooldown,

748
awx/lib/site-packages/pyrax/base_identity.py
View File

File diff suppressed because it is too large Load Diff

345
awx/lib/site-packages/pyrax/cf_wrapper/client.py
View File

@ -1,6 +1,23 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import logging
import datetime
from functools import wraps
import hashlib
@ -44,6 +61,7 @@ CONNECTION_TIMEOUT = 20
CONNECTION_RETRIES = 5
AUTH_ATTEMPTS = 2
MAX_BULK_DELETE = 10000
DEFAULT_CHUNKSIZE = 65536
no_such_container_pattern = re.compile(
r"Container (?:GET|HEAD) failed: .+/(.+) 404")
@ -60,6 +78,17 @@ def _close_swiftclient_conn(conn):
pass
def plug_hole_in_swiftclient_auth(clt, url):
"""
This is necessary because swiftclient has an issue when a token expires and
it needs to re-authenticate against Rackspace auth. It is a temporary
workaround until we can fix swiftclient.
"""
conn = clt.connection
conn.token = clt.identity.token
conn.url = url
def handle_swiftclient_exception(fnc):
@wraps(fnc)
def _wrapped(self, *args, **kwargs):
@ -79,9 +108,9 @@ def handle_swiftclient_exception(fnc):
# Assume it is an auth failure. Re-auth and retry.
# NOTE: This is a hack to get around an apparent bug
# in python-swiftclient when using Rackspace auth.
pyrax.authenticate(connect=False)
if pyrax.identity.authenticated:
pyrax.plug_hole_in_swiftclient_auth(self, clt_url)
self.identity.authenticate(connect=False)
if self.identity.authenticated:
self.plug_hole_in_swiftclient_auth(self, clt_url)
continue
elif e.http_status == 404:
bad_container = no_such_container_pattern.search(str_error)
@ -102,6 +131,16 @@ def handle_swiftclient_exception(fnc):
return _wrapped
def ensure_cdn(fnc):
@wraps(fnc)
def _wrapped(self, *args, **kwargs):
if not self.connection.cdn_connection:
raise exc.NotCDNEnabled("This service does not support "
"CDN-enabled containers.")
return fnc(self, *args, **kwargs)
return _wrapped
def _convert_head_object_last_modified_to_local(lm_str):
# Need to convert last modified time to a datetime object.
# Times are returned in default locale format, so we need to read
@ -119,18 +158,32 @@ def _convert_head_object_last_modified_to_local(lm_str):
def _convert_list_last_modified_to_local(attdict):
if 'last_modified' in attdict:
if "last_modified" in attdict:
attdict = attdict.copy()
list_date_format_with_tz = LIST_DATE_FORMAT + ' %Z'
last_modified_utc = attdict['last_modified'] + ' UTC'
list_date_format_with_tz = LIST_DATE_FORMAT + " %Z"
last_modified_utc = attdict["last_modified"] + " UTC"
tm_tuple = time.strptime(last_modified_utc,
list_date_format_with_tz)
dttm = datetime.datetime.fromtimestamp(time.mktime(tm_tuple))
attdict['last_modified'] = dttm.strftime(DATE_FORMAT)
dttm_with_micros = datetime.datetime.strptime(last_modified_utc,
list_date_format_with_tz)
# Round the date *up* in seconds, to match the last modified time
# in head requests
# https://review.openstack.org/#/c/55488/
if dttm_with_micros.microsecond > 0:
dttm += datetime.timedelta(seconds=1)
attdict["last_modified"] = dttm.strftime(DATE_FORMAT)
return attdict
def _quote(val):
if isinstance(val, six.text_type):
val = val.encode("utf-8")
return urllib.quote(val)
class CFClient(object):
"""
Wraps the calls to swiftclient with objects representing Containers
@ -139,11 +192,6 @@ class CFClient(object):
These classes allow a developer to work with regular Python objects
instead of calling functions that return primitive types.
"""
# Constants used in metadata headers
account_meta_prefix = "X-Account-Meta-"
container_meta_prefix = "X-Container-Meta-"
object_meta_prefix = "X-Object-Meta-"
cdn_meta_prefix = "X-Cdn-"
# Defaults for CDN
cdn_enabled = False
default_cdn_ttl = 86400
@ -175,6 +223,24 @@ class CFClient(object):
http_log_debug=http_log_debug)
# Constants used in metadata headers
@property
def account_meta_prefix(self):
return "X-Account-Meta-"
@property
def container_meta_prefix(self):
return "X-Container-Meta-"
@property
def object_meta_prefix(self):
return "X-Object-Meta-"
@property
def cdn_meta_prefix(self):
return "X-Cdn-"
def _make_connections(self, auth_endpoint, username, api_key, password,
tenant_name=None, preauthurl=None, preauthtoken=None,
auth_version="2", os_options=None, verify_ssl=True,
@ -210,12 +276,14 @@ class CFClient(object):
@handle_swiftclient_exception
def get_account_metadata(self):
def get_account_metadata(self, prefix=None):
headers = self.connection.head_account()
prfx = self.account_meta_prefix.lower()
if prefix is None:
prefix = self.account_meta_prefix
prefix = prefix.lower()
ret = {}
for hkey, hval in headers.iteritems():
if hkey.lower().startswith(prfx):
if hkey.lower().startswith(prefix):
ret[hkey] = hval
return ret
@ -336,11 +404,7 @@ class CFClient(object):
specified number of seconds.
"""
meta = {"X-Delete-After": str(seconds)}
self.set_object_metadata(cont, obj, meta, prefix="")
# cname = self._resolve_name(cont)
# oname = self._resolve_name(obj)
# self.connection.post_object(cname, oname, headers=headers,
# response_dict=extra_info)
self.set_object_metadata(cont, obj, meta, prefix="", clear=True)
@handle_swiftclient_exception
@ -349,7 +413,8 @@ class CFClient(object):
cname = self._resolve_name(container)
headers = self.connection.head_container(cname)
if prefix is None:
prefix = self.container_meta_prefix.lower()
prefix = self.container_meta_prefix
prefix = prefix.lower()
ret = {}
for hkey, hval in headers.iteritems():
if hkey.lower().startswith(prefix):
@ -394,19 +459,23 @@ class CFClient(object):
@handle_swiftclient_exception
def remove_container_metadata_key(self, container, key,
extra_info=None):
prefix=None, extra_info=None):
"""
Removes the specified key from the container's metadata. If the key
does not exist in the metadata, nothing is done.
"""
if prefix is None:
prefix = self.container_meta_prefix
prefix = prefix.lower()
meta_dict = {key: ""}
# Add the metadata prefix, if needed.
massaged = self._massage_metakeys(meta_dict, self.container_meta_prefix)
massaged = self._massage_metakeys(meta_dict, prefix)
cname = self._resolve_name(container)
self.connection.post_container(cname, massaged,
response_dict=extra_info)
@ensure_cdn
@handle_swiftclient_exception
def get_container_cdn_metadata(self, container):
"""
@ -421,6 +490,7 @@ class CFClient(object):
return dict(headers)
@ensure_cdn
@handle_swiftclient_exception
def set_container_cdn_metadata(self, container, metadata):
"""
@ -449,15 +519,17 @@ class CFClient(object):
@handle_swiftclient_exception
def get_object_metadata(self, container, obj):
def get_object_metadata(self, container, obj, prefix=None):
"""Retrieves any metadata for the specified object."""
if prefix is None:
prefix = self.object_meta_prefix
cname = self._resolve_name(container)
oname = self._resolve_name(obj)
headers = self.connection.head_object(cname, oname)
prfx = self.object_meta_prefix.lower()
prefix = prefix.lower()
ret = {}
for hkey, hval in headers.iteritems():
if hkey.lower().startswith(prfx):
if hkey.lower().startswith(prefix):
ret[hkey] = hval
return ret
@ -493,8 +565,8 @@ class CFClient(object):
# whereas for containers you need to set the values to an empty
# string to delete them.
if not clear:
obj_meta = self.get_object_metadata(cname, oname)
new_meta = self._massage_metakeys(obj_meta, self.object_meta_prefix)
obj_meta = self.get_object_metadata(cname, oname, prefix=prefix)
new_meta = self._massage_metakeys(obj_meta, prefix)
utils.case_insensitive_update(new_meta, massaged)
# Remove any empty values, since the object metadata API will
# store them.
@ -509,12 +581,12 @@ class CFClient(object):
@handle_swiftclient_exception
def remove_object_metadata_key(self, container, obj, key):
def remove_object_metadata_key(self, container, obj, key, prefix=None):
"""
Removes the specified key from the storage object's metadata. If the key
does not exist in the metadata, nothing is done.
Removes the specified key from the storage object's metadata. If the
key does not exist in the metadata, nothing is done.
"""
self.set_object_metadata(container, obj, {key: ""})
self.set_object_metadata(container, obj, {key: ""}, prefix=prefix)
@handle_swiftclient_exception
@ -629,33 +701,50 @@ class CFClient(object):
@handle_swiftclient_exception
def store_object(self, container, obj_name, data, content_type=None,
etag=None, content_encoding=None, ttl=None, return_none=False,
extra_info=None):
chunk_size=None, headers=None, extra_info=None):
"""
Creates a new object in the specified container, and populates it with
the given data. A StorageObject reference to the uploaded file
will be returned, unless 'return_none' is set to True.
'chunk_size' represents the number of bytes of data to write; it
defaults to 65536. It is used only if the the 'data' parameter is an
object with a 'read' method; otherwise, it is ignored.
If you wish to specify additional headers to be passed to the PUT
request, pass them as a dict in the 'headers' parameter. It is the
developer's responsibility to ensure that any headers are valid; pyrax
does no checking.
'extra_info' is an optional dictionary which will be
populated with 'status', 'reason', and 'headers' keys from the
underlying swiftclient call.
"""
cont = self.get_container(container)
headers = {}
if headers is None:
headers = {}
if content_encoding is not None:
headers["Content-Encoding"] = content_encoding
if ttl is not None:
headers["X-Delete-After"] = ttl
with utils.SelfDeletingTempfile() as tmp:
with open(tmp, "wb") as tmpfile:
try:
tmpfile.write(data)
except UnicodeEncodeError:
udata = data.encode("utf-8")
tmpfile.write(udata)
with open(tmp, "rb") as tmpfile:
self.connection.put_object(cont.name, obj_name,
contents=tmpfile, content_type=content_type, etag=etag,
headers=headers, response_dict=extra_info)
if chunk_size and hasattr(data, "read"):
# Chunked file-like object
self.connection.put_object(cont.name, obj_name, contents=data,
content_type=content_type, etag=etag, headers=headers,
chunk_size=chunk_size, response_dict=extra_info)
else:
with utils.SelfDeletingTempfile() as tmp:
with open(tmp, "wb") as tmpfile:
try:
tmpfile.write(data)
except UnicodeEncodeError:
udata = data.encode("utf-8")
tmpfile.write(udata)
with open(tmp, "rb") as tmpfile:
self.connection.put_object(cont.name, obj_name,
contents=tmpfile, content_type=content_type,
etag=etag, headers=headers, chunk_size=chunk_size,
response_dict=extra_info)
if return_none:
return None
else:
@ -721,7 +810,7 @@ class CFClient(object):
def upload_file(self, container, file_or_path, obj_name=None,
content_type=None, etag=None, return_none=False,
content_encoding=None, ttl=None, extra_info=None,
content_length=None):
content_length=None, headers=None):
"""
Uploads the specified file to the container. If no name is supplied,
the file's name will be used. Either a file path or an open file-like
@ -734,6 +823,11 @@ class CFClient(object):
If the size of the file is known, it can be passed as `content_length`.
If you wish to specify additional headers to be passed to the PUT
request, pass them as a dict in the 'headers' parameter. It is the
developer's responsibility to ensure that any headers are valid; pyrax
does no checking.
If you wish for the object to be temporary, specify the time it should
be stored in seconds in the `ttl` parameter. If this is specified, the
object will be deleted after that number of seconds.
@ -806,7 +900,8 @@ class CFClient(object):
raise InvalidUploadID("No filename provided and/or it cannot be "
"inferred from context")
headers = {}
if headers is None:
headers = {}
if content_encoding is not None:
headers["Content-Encoding"] = content_encoding
if ttl is not None:
@ -886,7 +981,8 @@ class CFClient(object):
def sync_folder_to_container(self, folder_path, container, delete=False,
include_hidden=False, ignore=None, ignore_timestamps=False):
include_hidden=False, ignore=None, ignore_timestamps=False,
object_prefix="", verbose=False):
"""
Compares the contents of the specified folder, and checks to make sure
that the corresponding object is present in the specified container. If
@ -909,22 +1005,43 @@ class CFClient(object):
file names, and any names that match any of the 'ignore' patterns will
not be uploaded. The patterns should be standard *nix-style shell
patterns; e.g., '*pyc' will ignore all files ending in 'pyc', such as
'program.pyc' and 'abcpyc'. """
'program.pyc' and 'abcpyc'.
If `object_prefix` is set it will be appended to the object name when
it is checked and uploaded to the container. For example, if you use
sync_folder_to_container("folderToSync/", myContainer,
object_prefix="imgFolder") it will upload the files to the
container/imgFolder/... instead of just container/...
Set `verbose` to True to make it print what is going on. It will
show which files are being uploaded and which ones are not and why.
"""
cont = self.get_container(container)
self._local_files = []
# Load a list of all the remote objects so we don't have to keep
# hitting the service
if verbose:
log = logging.getLogger("pyrax")
log.info("Loading remote object list (prefix=%s)", object_prefix)
data = cont.get_objects(prefix=object_prefix, full_listing=True)
self._remote_files = dict((d.name, d) for d in data)
self._sync_folder_to_container(folder_path, cont, prefix="",
delete=delete, include_hidden=include_hidden, ignore=ignore,
ignore_timestamps=ignore_timestamps)
ignore_timestamps=ignore_timestamps,
object_prefix=object_prefix, verbose=verbose)
# Unset the _remote_files
self._remote_files = None
def _sync_folder_to_container(self, folder_path, cont, prefix, delete,
include_hidden, ignore, ignore_timestamps):
include_hidden, ignore, ignore_timestamps, object_prefix, verbose):
"""
This is the internal method that is called recursively to handle
nested folder structures.
"""
fnames = os.listdir(folder_path)
ignore = utils.coerce_string_to_list(ignore)
log = logging.getLogger("pyrax")
if not include_hidden:
ignore.append(".*")
for fname in fnames:
@ -937,17 +1054,20 @@ class CFClient(object):
subprefix = "%s/%s" % (prefix, subprefix)
self._sync_folder_to_container(pth, cont, prefix=subprefix,
delete=delete, include_hidden=include_hidden,
ignore=ignore, ignore_timestamps=ignore_timestamps)
ignore=ignore, ignore_timestamps=ignore_timestamps,
object_prefix=object_prefix, verbose=verbose)
continue
self._local_files.append(os.path.join(prefix, fname))
self._local_files.append(os.path.join(object_prefix, prefix, fname))
local_etag = utils.get_checksum(pth)
fullname = fname
fullname_with_prefix = "%s/%s" % (object_prefix, fname)
if prefix:
fullname = "%s/%s" % (prefix, fname)
fullname_with_prefix = "%s/%s/%s" % (object_prefix, prefix, fname)
try:
obj = cont.get_object(fullname)
obj = self._remote_files[fullname_with_prefix]
obj_etag = obj.etag
except exc.NoSuchObject:
except KeyError:
obj = None
obj_etag = None
if local_etag != obj_etag:
@ -961,19 +1081,29 @@ class CFClient(object):
local_mod_str = local_mod.isoformat()
if obj_time_str >= local_mod_str:
# Remote object is newer
if verbose:
log.info("%s NOT UPLOADED because remote object is "
"newer", fullname)
continue
cont.upload_file(pth, obj_name=fullname, etag=local_etag,
return_none=True)
cont.upload_file(pth, obj_name=fullname_with_prefix,
etag=local_etag, return_none=True)
if verbose:
log.info("%s UPLOADED", fullname)
else:
if verbose:
log.info("%s NOT UPLOADED because it already exists",
fullname)
if delete and not prefix:
self._delete_objects_not_in_list(cont)
self._delete_objects_not_in_list(cont, object_prefix)
def _delete_objects_not_in_list(self, cont):
def _delete_objects_not_in_list(self, cont, object_prefix=""):
"""
Finds all the objects in the specified container that are not present
in the self._local_files list, and deletes them.
"""
objnames = set(cont.get_object_names(full_listing=True))
objnames = set(cont.get_object_names(prefix=object_prefix,
full_listing=True))
localnames = set(self._local_files)
to_delete = list(objnames.difference(localnames))
# We don't need to wait around for this to complete. Store the thread
@ -1067,6 +1197,52 @@ class CFClient(object):
return ret
def fetch_dlo(self, cont, name, chunk_size=None):
"""
Returns a list of 2-tuples in the form of (object_name,
fetch_generator) representing the components of a multi-part DLO
(Dynamic Large Object). Each fetch_generator object can be interated
to retrieve its contents.
This is useful when transferring a DLO from one object storage system
to another. Examples would be copying DLOs from one region of a
provider to another, or copying a DLO from one provider to another.
"""
if chunk_size is None:
chunk_size = DEFAULT_CHUNKSIZE
class FetchChunker(object):
"""
Class that takes the generator objects returned by a chunked
fetch_object() call and wraps them to behave as file-like objects for
uploading.
"""
def __init__(self, gen, verbose=False):
self.gen = gen
self.verbose = verbose
self.processed = 0
self.interval = 0
def read(self, size=None):
self.interval += 1
if self.verbose:
if self.interval > 1024:
self.interval = 0
logit(".")
ret = self.gen.next()
self.processed += len(ret)
return ret
parts = self.get_container_objects(cont, prefix=name)
fetches = [(part.name, self.fetch_object(cont, part.name,
chunk_size=chunk_size))
for part in parts
if part.name != name]
job = [(fetch[0], FetchChunker(fetch[1], verbose=False))
for fetch in fetches]
return job
@handle_swiftclient_exception
def download_object(self, container, obj, directory, structure=True):
"""
@ -1096,7 +1272,8 @@ class CFClient(object):
@handle_swiftclient_exception
def get_all_containers(self, limit=None, marker=None, **parms):
hdrs, conts = self.connection.get_container("")
hdrs, conts = self.connection.get_container("", limit=limit,
marker=marker)
ret = [Container(self, name=cont["name"], object_count=cont["count"],
total_bytes=cont["bytes"]) for cont in conts]
return ret
@ -1123,6 +1300,7 @@ class CFClient(object):
total_bytes=hdrs.get("x-container-bytes-used"))
self._container_cache[cname] = cont
return cont
get = get_container
@handle_swiftclient_exception
@ -1145,6 +1323,7 @@ class CFClient(object):
attdict=_convert_list_last_modified_to_local(obj))
for obj in objs
if "name" in obj]
list_container_objects = get_container_objects
@handle_swiftclient_exception
@ -1156,24 +1335,26 @@ class CFClient(object):
full_listing=full_listing)
cont = self.get_container(cname)
return [obj["name"] for obj in objs]
list_container_object_names = get_container_object_names
@handle_swiftclient_exception
def list_container_subdirs(self, container, marker=None, limit=None,
prefix=None, delimiter=None, full_listing=False):
"""
Return a list of StorageObjects representing the pseudo-subdirectories
Returns a list of StorageObjects representing the pseudo-subdirectories
in the specified container. You can use the marker and limit params to
handle pagination, and the prefix and delimiter params to filter the
objects returned.
handle pagination, and the prefix param to filter the objects returned.
The 'delimiter' parameter is ignored, as the only meaningful value is
'/'.
"""
cname = self._resolve_name(container)
hdrs, objs = self.connection.get_container(cname, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
limit=limit, prefix=prefix, delimiter="/",
full_listing=full_listing)
cont = self.get_container(cname)
return [StorageObject(self, container=cont, attdict=obj) for obj in objs
if obj.get("content_type") == "application/directory"]
if "subdir" in obj]
@handle_swiftclient_exception
@ -1189,17 +1370,21 @@ class CFClient(object):
@handle_swiftclient_exception
def list(self, limit=None, marker=None, **parms):
"""Returns a list of all container objects."""
hdrs, conts = self.connection.get_container("")
hdrs, conts = self.connection.get_container("", limit=limit,
marker=marker)
ret = [self.get_container(cont["name"]) for cont in conts]
return ret
get_all_containers = list
@handle_swiftclient_exception
def list_containers(self, limit=None, marker=None, **parms):
"""Returns a list of all container names as strings."""
hdrs, conts = self.connection.get_container("")
hdrs, conts = self.connection.get_container("", limit=limit,
marker=marker)
ret = [cont["name"] for cont in conts]
return ret
list_container_names = list_containers
@handle_swiftclient_exception
@ -1212,10 +1397,12 @@ class CFClient(object):
count - the number of objects in the container
bytes - the total bytes in the container
"""
hdrs, conts = self.connection.get_container("")
hdrs, conts = self.connection.get_container("", limit=limit,
marker=marker)
return conts
@ensure_cdn
@handle_swiftclient_exception
def list_public_containers(self):
"""Returns a list of all CDN-enabled containers."""
@ -1240,6 +1427,7 @@ class CFClient(object):
return self._cdn_set_access(container, None, False)
@ensure_cdn
def _cdn_set_access(self, container, ttl, enabled):
"""Used to enable or disable CDN access on a container."""
if ttl is None:
@ -1273,6 +1461,7 @@ class CFClient(object):
cont.cdn_log_retention = enabled
@ensure_cdn
def _set_cdn_log_retention(self, container, enabled):
"""This does the actual call to the Cloud Files API."""
hdrs = {"X-Log-Retention": "%s" % enabled}
@ -1324,6 +1513,7 @@ class CFClient(object):
return self.set_container_metadata(container, hdr, clear=False)
@ensure_cdn
@handle_swiftclient_exception
def purge_cdn_object(self, container, name, email_addresses=None):
ct = self.get_container(container)
@ -1413,12 +1603,7 @@ class Connection(_swift_client.Connection):
Taken directly from the cloudfiles library and modified for use here.
"""
def quote(val):
if isinstance(val, six.text_type):
val = val.encode("utf-8")
return urllib.quote(val)
pth = "/".join([quote(elem) for elem in path])
pth = "/".join([_quote(elem) for elem in path])
uri_path = urlparse.urlparse(self.uri).path
path = "%s/%s" % (uri_path.rstrip("/"), pth)
headers = {"Content-Length": str(len(data)),
@ -1440,8 +1625,8 @@ class Connection(_swift_client.Connection):
response = None
if response:
if response.status == 401:
pyrax.identity.authenticate()
headers["X-Auth-Token"] = pyrax.identity.token
self.identity.authenticate()
headers["X-Auth-Token"] = self.identity.token
else:
break
attempt += 1
@ -1532,14 +1717,14 @@ class BulkDeleter(threading.Thread):
cname = client._resolve_name(container)
parsed, conn = client.connection.http_connection()
method = "DELETE"
headers = {"X-Auth-Token": pyrax.identity.token,
headers = {"X-Auth-Token": self.client.identity.token,
"Content-type": "text/plain",
}
while object_names:
this_batch, object_names = (object_names[:MAX_BULK_DELETE],
object_names[MAX_BULK_DELETE:])
obj_paths = ("%s/%s" % (cname, nm) for nm in this_batch)
body = "\n".join(obj_paths)
body = _quote("\n".join(obj_paths))
pth = "%s/?bulk-delete=1" % parsed.path
conn.request(method, pth, body, headers)
resp = conn.getresponse()
@ -1547,7 +1732,7 @@ class BulkDeleter(threading.Thread):
reason = resp.reason
resp_body = resp.read()
for resp_line in resp_body.splitlines():
if not resp_line:
if not resp_line.strip():
continue
resp_key, val = resp_line.split(":", 1)
result_key = res_keys.get(resp_key)

18
awx/lib/site-packages/pyrax/cf_wrapper/container.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -56,6 +56,10 @@ class Container(object):
def _fetch_cdn_data(self):
"""Fetches the object's CDN data from the CDN service"""
if not self.client.cdn_enabled:
# Not CDN enabled; set the defaults.
self._set_cdn_defaults()
return
response = self.client.connection.cdn_request("HEAD", [self.name])
if 200 <= response.status < 300:
# Set defaults in case not all headers are present.
@ -95,6 +99,7 @@ class Container(object):
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
return objs
list = get_objects
def get_object(self, name, cached=True):
@ -113,6 +118,7 @@ class Container(object):
ret = self.client.get_object(self, name)
self._object_cache[name] = ret
return ret
get = get_object
def get_object_names(self, marker=None, limit=None, prefix=None,
@ -124,6 +130,7 @@ class Container(object):
return self.client.get_container_object_names(self.name, marker=marker,
limit=limit, prefix=prefix, delimiter=delimiter,
full_listing=full_listing)
list_object_names = get_object_names
def list_subdirs(self, marker=None, limit=None, prefix=None, delimiter=None,
@ -244,11 +251,11 @@ class Container(object):
structure=structure)
def get_metadata(self):
def get_metadata(self, prefix=None):
"""
Returns a dictionary containing the metadata for the container.
"""
return self.client.get_container_metadata(self)
return self.client.get_container_metadata(self, prefix=prefix)
def set_metadata(self, metadata, clear=False, prefix=None):
@ -273,12 +280,13 @@ class Container(object):
prefix=prefix)
def remove_metadata_key(self, key):
def remove_metadata_key(self, key, prefix=None):
"""
Removes the specified key from the container's metadata. If the key
does not exist in the metadata, nothing is done.
"""
return self.client.remove_container_metadata_key(self, key)
return self.client.remove_container_metadata_key(self, key,
prefix=prefix)
def set_web_index_page(self, page):

12
awx/lib/site-packages/pyrax/cf_wrapper/storage_object.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -106,9 +106,10 @@ class StorageObject(object):
name=self.name, email_addresses=email_addresses)
def get_metadata(self):
def get_metadata(self, prefix=None):
"""Returns this object's metadata."""
return self.client.get_object_metadata(self.container, self)
return self.client.get_object_metadata(self.container, self,
prefix=prefix)
def set_metadata(self, metadata, clear=False, prefix=None):
@ -119,12 +120,13 @@ class StorageObject(object):
clear=clear, prefix=prefix)
def remove_metadata_key(self, key):
def remove_metadata_key(self, key, prefix=None):
"""
Removes the specified key from the storage object's metadata. If the
key does not exist in the metadata, nothing is done.
"""
self.client.remove_object_metadata_key(self.container, self, key)
self.client.remove_object_metadata_key(self.container, self, key,
prefix=prefix)
def copy(self, new_container, new_obj_name=None, extra_info=None):

47
awx/lib/site-packages/pyrax/client.py
View File

@ -1,7 +1,10 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2010 Jacob Kaplan-Moss
# Copyright 2011 OpenStack LLC.
# Copyright 2011 Piston Cloud Computing, Inc.
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -20,17 +23,30 @@
OpenStack Client interface. Handles the REST calls and responses.
"""
from __future__ import absolute_import
import json
import logging
import requests
import time
import urllib
import urlparse
import six.moves.urllib as urllib
import pyrax
import pyrax.exceptions as exc
def _safe_quote(val):
"""
Unicode values will raise a KeyError, so catch those and encode in UTF-8.
"""
SAFE_QUOTE_CHARS = "/.?&=,"
try:
ret = urllib.parse.quote(val, safe=SAFE_QUOTE_CHARS)
except KeyError:
ret = urllib.parse.quote(val.encode("utf-8"), safe=SAFE_QUOTE_CHARS)
return ret
class BaseClient(object):
"""
The base class for all pyrax clients.
@ -40,10 +56,11 @@ class BaseClient(object):
# Each client subclass should set their own name.
name = "base"
def __init__(self, region_name=None, endpoint_type="publicURL",
def __init__(self, identity, region_name=None, endpoint_type=None,
management_url=None, service_name=None, timings=False,
verify_ssl=True, http_log_debug=False, timeout=None):
self.version = "v1.1"
self.identity = identity
self.region_name = region_name
self.endpoint_type = endpoint_type
self.service_name = service_name
@ -114,7 +131,7 @@ class BaseClient(object):
def unauthenticate(self):
"""Clears all of our authentication information."""
pyrax.identity.unauthenticate()
self.identity.unauthenticate()
def get_timings(self):
@ -158,6 +175,11 @@ class BaseClient(object):
kwargs.setdefault("headers", kwargs.get("headers", {}))
kwargs["headers"]["User-Agent"] = self.user_agent
kwargs["headers"]["Accept"] = "application/json"
if ("body" in kwargs) or ("data" in kwargs):
if "Content-Type" not in kwargs["headers"]:
kwargs["headers"]["Content-Type"] = "application/json"
elif kwargs["headers"]["Content-Type"] is None:
del kwargs["headers"]["Content-Type"]
# Allow subclasses to add their own headers
self._add_custom_headers(kwargs["headers"])
resp, body = pyrax.http.request(method, uri, *args, **kwargs)
@ -181,7 +203,7 @@ class BaseClient(object):
the request after authenticating if the initial request returned
and Unauthorized exception.
"""
id_svc = pyrax.identity
id_svc = self.identity
if not all((self.management_url, id_svc.token, id_svc.tenant_id)):
id_svc.authenticate()
@ -191,16 +213,15 @@ class BaseClient(object):
raise exc.ServiceNotAvailable("The '%s' service is not available."
% self)
if uri.startswith("http"):
parsed = list(urlparse.urlparse(uri))
parsed = list(urllib.parse.urlparse(uri))
for pos, item in enumerate(parsed):
if pos < 2:
# Don't escape the scheme or netloc
continue
parsed[pos] = urllib.quote(parsed[pos], safe="/.?&=,")
safe_uri = urlparse.urlunparse(parsed)
parsed[pos] = _safe_quote(parsed[pos])
safe_uri = urllib.parse.urlunparse(parsed)
else:
safe_uri = "%s%s" % (self.management_url,
urllib.quote(uri, safe="/.?&=,"))
safe_uri = "%s%s" % (self.management_url, _safe_quote(uri))
# Perform the request once. If we get a 401 back then it
# might be because the auth token expired, so try to
# re-authenticate and try again. If it still fails, bail.
@ -258,7 +279,7 @@ class BaseClient(object):
to modify this method. Please post your findings on GitHub so that
others can benefit.
"""
return pyrax.identity.authenticate()
return self.identity.authenticate()
@property
@ -267,4 +288,4 @@ class BaseClient(object):
The older parts of this code used 'projectid'; this wraps that
reference.
"""
return pyrax.identity.tenant_id
return self.identity.tenant_id

71
awx/lib/site-packages/pyrax/cloudblockstorage.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -27,6 +27,7 @@ from pyrax.client import BaseClient
import pyrax.exceptions as exc
from pyrax.manager import BaseManager
from pyrax.resource import BaseResource
import pyrax.utils as utils
MIN_SIZE = 100
@ -129,7 +130,9 @@ class CloudBlockStorageVolume(BaseResource):
def __init__(self, *args, **kwargs):
super(CloudBlockStorageVolume, self).__init__(*args, **kwargs)
region = self.manager.api.region_name
self._nova_volumes = pyrax.connect_to_cloudservers(region).volumes
context = self.manager.api.identity
cs = pyrax.connect_to_cloudservers(region=region, context=context)
self._nova_volumes = cs.volumes
def attach_to_instance(self, instance, mountpoint):
@ -286,6 +289,24 @@ class CloudBlockStorageManager(BaseManager):
raise
def update(self, volume, display_name=None, display_description=None):
"""
Update the specified values on the specified volume. You may specify
one or more values to update.
"""
uri = "/%s/%s" % (self.uri_base, utils.get_id(volume))
param_dict = {}
if display_name:
param_dict["display_name"] = display_name
if display_description:
param_dict["display_description"] = display_description
if not param_dict:
# Nothing to do!
return
body = {"volume": param_dict}
resp, resp_body = self.api.method_put(uri, body=body)
def list_snapshots(self):
"""
Pass-through method to allow the list_snapshots() call to be made
@ -354,6 +375,24 @@ class CloudBlockStorageSnapshotManager(BaseManager):
return snap
def update(self, snapshot, display_name=None, display_description=None):
"""
Update the specified values on the specified snapshot. You may specify
one or more values to update.
"""
uri = "/%s/%s" % (self.uri_base, utils.get_id(snapshot))
param_dict = {}
if display_name:
param_dict["display_name"] = display_name
if display_description:
param_dict["display_description"] = display_description
if not param_dict:
# Nothing to do!
return
body = {"snapshot": param_dict}
resp, resp_body = self.api.method_put(uri, body=body)
class CloudBlockStorageClient(BaseClient):
"""
This is the primary class for interacting with Cloud Block Storage.
@ -404,6 +443,16 @@ class CloudBlockStorageClient(BaseClient):
return volume.delete(force=force)
@assure_volume
def update(self, volume, display_name=None, display_description=None):
"""
Update the specified values on the specified volume. You may specify
one or more values to update.
"""
return self._manager.update(volume, display_name=display_name,
display_description=display_description)
@assure_volume
def create_snapshot(self, volume, name=None, description=None, force=False):
"""
@ -416,7 +465,25 @@ class CloudBlockStorageClient(BaseClient):
description=description, force=force)
def get_snapshot(self, snapshot):
"""
Returns the snapshot with the specified snapshot ID value.
"""
return self._snapshot_manager.get(snapshot)
@assure_snapshot
def delete_snapshot(self, snapshot):
"""Deletes the snapshot."""
return snapshot.delete()
def update_snapshot(self, snapshot, display_name=None,
display_description=None):
"""
Update the specified values on the specified snapshot. You may specify
one or more values to update.
"""
return self._snapshot_manager.update(snapshot,
display_name=display_name,
display_description=display_description)

2
awx/lib/site-packages/pyrax/clouddatabases.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#

8
awx/lib/site-packages/pyrax/clouddns.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -871,12 +871,14 @@ class CloudDNSManager(BaseManager):
Takes a device and device type and returns the corresponding HREF link
and service name for use with PTR record management.
"""
context = self.api.identity
region = self.api.region_name
if device_type.lower().startswith("load"):
ep = pyrax._get_service_endpoint("load_balancer")
ep = pyrax._get_service_endpoint(context, "load_balancer", region)
svc = "loadbalancers"
svc_name = "cloudLoadBalancers"
else:
ep = pyrax._get_service_endpoint("compute")
ep = pyrax._get_service_endpoint(context, "compute", region)
svc = "servers"
svc_name = "cloudServersOpenStack"
href = "%s/%s/%s" % (ep, svc, utils.get_id(device))

12
awx/lib/site-packages/pyrax/cloudloadbalancers.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -85,9 +85,15 @@ class CloudLoadBalancer(BaseResource):
return self.manager.get_usage(self, start=start, end=end)
def get_stats(self):
"""
Return the stats for this loadbalancer
"""
return self.manager.get_stats(self)
def _add_details(self, info):
"""Override the base behavior to add Nodes, VirtualIPs, etc."""
for (key, val) in info.iteritems():
for (key, val) in six.iteritems(info):
if key == "nodes":
val = [Node(parent=self, **nd) for nd in val]
elif key == "sessionPersistence":
@ -956,7 +962,7 @@ class CloudLoadBalancerManager(BaseManager):
return body
def get_stats(self, loadbalancer):
def get_stats(self, loadbalancer=None):
"""
Returns statistics for the given load balancer.
"""

23
awx/lib/site-packages/pyrax/cloudmonitoring.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Rackspace
# Copyright (c)2013 Rackspace US, Inc.
# All Rights Reserved.
#
@ -162,7 +162,7 @@ class CloudMonitorNotificationManager(BaseManager):
"details": details,
}
resp, resp_body = self.api.method_post(uri, body=body)
return self.get(resp["x-object-id"])
return self.get(resp.headers["x-object-id"])
def test_notification(self, notification=None, notification_type=None,
@ -259,7 +259,7 @@ class CloudMonitorNotificationPlanManager(BaseManager):
ok_state = utils.coerce_string_to_list(ok_state)
body["ok_state"] = make_list_of_ids(ok_state)
resp, resp_body = self.api.method_post(uri, body=body)
return self.get(resp["x-object-id"])
return self.get(resp.headers["x-object-id"])
class CloudMonitorEntityManager(BaseManager):
@ -380,9 +380,8 @@ class CloudMonitorEntityManager(BaseManager):
raise exc.InvalidMonitoringCheckDetails("Validation "
"failed. Error: '%s'." % dtls)
else:
status = resp["status"]
if status == "201":
check_id = resp["x-object-id"]
if resp.status_code == 201:
check_id = resp.headers["x-object-id"]
return self.get_check(entity, check_id)
@ -564,12 +563,11 @@ class CloudMonitorEntityManager(BaseManager):
if metadata:
body["metadata"] = metadata
resp, resp_body = self.api.method_post(uri, body=body)
status = resp["status"]
if status == "201":
alarm_id = resp["x-object-id"]
if resp.status_code == 201:
alarm_id = resp.headers["x-object-id"]
return self.get_alarm(entity, alarm_id)
def update_alarm(self, entity, alarm, criteria=None, disabled=False,
label=None, name=None, metadata=None):
"""
@ -948,9 +946,8 @@ class CloudMonitorClient(BaseClient):
resp = self._entity_manager.create(label=label, name=name, agent=agent,
ip_addresses=ip_addresses, metadata=metadata,
return_response=True)
status = resp["status"]
if status == "201":
ent_id = resp["x-object-id"]
if resp.status_code == 201:
ent_id = resp.headers["x-object-id"]
return self.get_entity(ent_id)

2
awx/lib/site-packages/pyrax/cloudnetworks.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2013 Rackspace
# Copyright (c)2013 Rackspace US, Inc.
# All Rights Reserved.
#

19
awx/lib/site-packages/pyrax/exceptions.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -245,6 +245,18 @@ class NetworkNotFound(PyraxException):
class NetworkLabelNotUnique(PyraxException):
pass
class NoClientForService(PyraxException):
pass
class NoEndpointForRegion(PyraxException):
pass
class NoEndpointForService(PyraxException):
pass
class NoContentSpecified(PyraxException):
pass
class NoMoreResults(PyraxException):
pass
@ -254,6 +266,9 @@ class NoReloadError(PyraxException):
class NoSSLTerminationConfiguration(PyraxException):
pass
class NoSuchClient(PyraxException):
pass
class NoSuchContainer(PyraxException):
pass
@ -466,6 +481,8 @@ def from_response(response, body):
else:
message = error
details = None
else:
message = body
return cls(code=status, message=message, details=details,
request_id=request_id)
else:

191
awx/lib/site-packages/pyrax/fakes.py
View File

@ -12,10 +12,6 @@ from pyrax.autoscale import AutoScalePolicy
from pyrax.autoscale import AutoScaleWebhook
from pyrax.autoscale import ScalingGroup
from pyrax.autoscale import ScalingGroupManager
from pyrax.cf_wrapper.client import BulkDeleter
from pyrax.cf_wrapper.client import FolderUploader
from pyrax.cf_wrapper.container import Container
from pyrax.cf_wrapper.storage_object import StorageObject
from pyrax.client import BaseClient
from pyrax.clouddatabases import CloudDatabaseClient
from pyrax.clouddatabases import CloudDatabaseDatabaseManager
@ -49,6 +45,13 @@ from pyrax.image import ImageClient
from pyrax.image import ImageManager
from pyrax.image import ImageMemberManager
from pyrax.image import ImageTagManager
from pyrax.object_storage import BulkDeleter
from pyrax.object_storage import Container
from pyrax.object_storage import ContainerManager
from pyrax.object_storage import FolderUploader
from pyrax.object_storage import StorageClient
from pyrax.object_storage import StorageObject
from pyrax.object_storage import StorageObjectManager
from pyrax.queueing import Queue
from pyrax.queueing import QueueClaim
from pyrax.queueing import QueueMessage
@ -56,6 +59,9 @@ from pyrax.queueing import QueueClient
from pyrax.queueing import QueueManager
import pyrax.exceptions as exc
from pyrax.base_identity import BaseIdentity
from pyrax.base_identity import Endpoint
from pyrax.base_identity import Service
from pyrax.identity.rax_identity import RaxIdentity
from pyrax.identity.keystone_identity import KeystoneIdentity
import pyrax.utils as utils
@ -88,41 +94,72 @@ class FakeResponse(object):
return "Line1\nLine2"
def get(self, arg):
pass
return self.headers.get(arg)
def json(self):
return self.content
class FakeIterator(utils.ResultsIterator):
def _init_methods(self):
pass
class FakeClient(object):
user_agent = "Fake"
USER_AGENT = "Fake"
def __init__(self, *args, **kwargs):
self.identity = FakeIdentity()
class FakeStorageClient(StorageClient):
def __init__(self, identity=None, *args, **kwargs):
if identity is None:
identity = FakeIdentity()
super(FakeStorageClient, self).__init__(identity, *args, **kwargs)
def create(self, name):
return FakeContainer(self._manager, {"name": name})
class FakeContainerManager(ContainerManager):
def __init__(self, api=None, *args, **kwargs):
if api is None:
api = FakeStorageClient()
super(FakeContainerManager, self).__init__(api, *args, **kwargs)
class FakeContainer(Container):
def _fetch_cdn_data(self):
self._cdn_uri = None
self._cdn_ttl = self.client.default_cdn_ttl
self._cdn_ssl_uri = None
self._cdn_streaming_uri = None
self._cdn_ios_uri = None
self._cdn_log_retention = False
def __init__(self, *args, **kwargs):
super(FakeContainer, self).__init__(*args, **kwargs)
self.object_manager = FakeStorageObjectManager(self.manager.api,
uri_base=self.name)
self.object_manager._container = self
class FakeStorageObjectManager(StorageObjectManager):
def __init__(self, api=None, *args, **kwargs):
if api is None:
api = FakeStorageClient()
if "uri_base" not in kwargs:
kwargs["uri_base"] = utils.random_ascii()
super(FakeStorageObjectManager, self).__init__(api, *args, **kwargs)
class FakeStorageObject(StorageObject):
def __init__(self, client, container, name=None, total_bytes=None,
content_type=None, last_modified=None, etag=None, attdict=None):
def __init__(self, manager, name=None, total_bytes=None, content_type=None,
last_modified=None, etag=None, attdict=None):
"""
The object can either be initialized with individual params, or by
passing the dict that is returned by swiftclient.
"""
self.client = client
self.container = container
self.manager = manager
self.name = name
self.total_bytes = total_bytes
self.bytes = total_bytes or 0
self.content_type = content_type
self.last_modified = last_modified
self.etag = etag
self.hash = etag
if attdict:
self._read_attdict(attdict)
@ -165,7 +202,8 @@ class FakeService(object):
class FakeCSClient(FakeService):
def __init__(self, *args, **kwargs):
super(FakeCSClient, self).__init__(*args, **kwargs)
ident = FakeIdentity()
super(FakeCSClient, self).__init__(ident, *args, **kwargs)
def dummy(self):
pass
@ -202,21 +240,10 @@ class FakeBulkDeleter(BulkDeleter):
self.completed = True
class FakeEntryPoint(object):
def __init__(self, name):
self.name = name
def load(self):
def dummy(*args, **kwargs):
return self.name
return dummy
fakeEntryPoints = [FakeEntryPoint("a"), FakeEntryPoint("b"),
FakeEntryPoint("c")]
class FakeManager(object):
api = FakeClient()
def __init__(self, *args, **kwargs):
super(FakeManager, self).__init__(*args, **kwargs)
self.api = FakeClient()
def list(self):
pass
@ -241,25 +268,6 @@ class FakeException(BaseException):
pass
class FakeServiceCatalog(object):
def __init__(self, *args, **kwargs):
pass
def get_token(self):
return "fake_token"
def url_for(self, attr=None, filter_value=None,
service_type=None, endpoint_type="publicURL",
service_name=None, volume_service_name=None):
if filter_value == "ALL":
raise exc.AmbiguousEndpoints
elif filter_value == "KEY":
raise KeyError
elif filter_value == "EP":
raise exc.EndpointNotFound
return "http://example.com"
class FakeKeyring(object):
password_set = False
@ -315,7 +323,8 @@ class FakeDatabaseClient(CloudDatabaseClient):
def __init__(self, *args, **kwargs):
self._manager = FakeDatabaseManager(self)
self._flavor_manager = FakeManager()
super(FakeDatabaseClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeDatabaseClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -326,8 +335,9 @@ class FakeNovaVolumeClient(BaseClient):
class FakeBlockStorageManager(CloudBlockStorageManager):
def __init__(self, api=None, *args, **kwargs):
ident = FakeIdentity()
if api is None:
api = FakeBlockStorageClient()
api = FakeBlockStorageClient(ident)
super(FakeBlockStorageManager, self).__init__(api, *args, **kwargs)
@ -350,13 +360,15 @@ class FakeBlockStorageClient(CloudBlockStorageClient):
def __init__(self, *args, **kwargs):
self._types_manager = FakeManager()
self._snapshot_manager = FakeManager()
super(FakeBlockStorageClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeBlockStorageClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
class FakeLoadBalancerClient(CloudLoadBalancerClient):
def __init__(self, *args, **kwargs):
super(FakeLoadBalancerClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeLoadBalancerClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -409,7 +421,8 @@ class FakeStatusChanger(object):
class FakeDNSClient(CloudDNSClient):
def __init__(self, *args, **kwargs):
super(FakeDNSClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeDNSClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -447,7 +460,8 @@ class FakeDNSDevice(FakeLoadBalancer):
class FakeCloudNetworkClient(CloudNetworkClient):
def __init__(self, *args, **kwargs):
super(FakeCloudNetworkClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeCloudNetworkClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -463,8 +477,9 @@ class FakeCloudNetwork(CloudNetwork):
class FakeAutoScaleClient(AutoScaleClient):
def __init__(self, *args, **kwargs):
ident = FakeIdentity()
self._manager = FakeManager()
super(FakeAutoScaleClient, self).__init__(*args, **kwargs)
super(FakeAutoScaleClient, self).__init__(ident, *args, **kwargs)
class FakeAutoScalePolicy(AutoScalePolicy):
@ -500,7 +515,8 @@ class FakeScalingGroup(ScalingGroup):
class FakeCloudMonitorClient(CloudMonitorClient):
def __init__(self, *args, **kwargs):
super(FakeCloudMonitorClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeCloudMonitorClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -547,20 +563,10 @@ class FakeQueueClaim(QueueClaim):
**kwargs)
class FakeQueueMessage(QueueMessage):
def __init__(self, *args, **kwargs):
id_ = utils.random_unicode()
href = "http://example.com/%s" % id_
info = kwargs.pop("info", {"href": href})
info["name"] = utils.random_unicode()
mgr = kwargs.pop("manager", FakeQueueManager())
super(FakeQueueMessage, self).__init__(manager=mgr, info=info, *args,
**kwargs)
class FakeQueueClient(QueueClient):
def __init__(self, *args, **kwargs):
super(FakeQueueClient, self).__init__("fakeuser",
ident = FakeIdentity()
super(FakeQueueClient, self).__init__(ident, "fakeuser",
"fakepassword", *args, **kwargs)
@ -584,8 +590,10 @@ class FakeImage(Image):
class FakeImageClient(ImageClient):
def __init__(self, *args, **kwargs):
super(FakeImageClient, self).__init__("fakeuser",
def __init__(self, identity=None, *args, **kwargs):
if identity is None:
identity = FakeIdentity()
super(FakeImageClient, self).__init__(identity, "fakeuser",
"fakepassword", *args, **kwargs)
@ -615,15 +623,43 @@ class FakeImageManager(ImageManager):
self.id = utils.random_ascii()
class FakeIdentity(RaxIdentity):
class FakeIdentityService(Service):
def __init__(self, identity=None, *args, **kwargs):
self.identity = identity or FakeIdentity()
self.name = "fake"
self.prefix = ""
self.service_type = "fake"
self.clients = {}
self.endpoints = utils.DotDict()
class FakeEndpoint(Endpoint):
def __init__(self, ep_dict=None, service=None, region=None, identity=None):
if ep_dict is None:
ep_dict = {}
if identity is None:
identity = FakeIdentity()
if service is None:
service = FakeIdentityService(identity)
if region is None:
region = "fake_region"
super(FakeEndpoint, self).__init__(ep_dict, service, region, identity)
class FakeRaxIdentity(RaxIdentity):
pass
class FakeIdentity(BaseIdentity):
"""Class that returns canned authentication responses."""
def __init__(self, *args, **kwargs):
super(FakeIdentity, self).__init__(*args, **kwargs)
self._good_username = "fakeuser"
self._good_password = "fakeapikey"
self._default_region = random.choice(("DFW", "ORD"))
self.services = {"fake": FakeIdentityService(self)}
def authenticate(self):
def authenticate(self, connect=False):
if ((self.username == self._good_username) and
(self.password == self._good_password)):
self._parse_response(self.fake_response())
@ -808,6 +844,9 @@ fake_identity_response = {u'access':
'region': 'DFW',
'tenantId': 'MossoCloudFS_abc'},
{u'publicURL': 'https://cdn1.clouddrive.com/v1/MossoCloudFS_abc',
'region': 'FAKE',
'tenantId': 'MossoCloudFS_abc'},
{u'publicURL': 'https://cdn1.clouddrive.com/v1/MossoCloudFS_abc',
'region': 'SYD',
'tenantId': 'MossoCloudFS_abc'},
{u'publicURL': 'https://cdn2.clouddrive.com/v1/MossoCloudFS_abc',

29
awx/lib/site-packages/pyrax/http.py
View File

@ -1,4 +1,4 @@
# Copyright 2014 Rackspace
# Copyright (c)2014 Rackspace US, Inc.
# All Rights Reserved.
#
@ -17,6 +17,7 @@
Wrapper around the requests library. Used for making all HTTP calls.
"""
import logging
import json
import requests
@ -48,7 +49,7 @@ def request(method, uri, *args, **kwargs):
req_method = req_methods[method.upper()]
raise_exception = kwargs.pop("raise_exception", True)
kwargs["headers"] = kwargs.get("headers", {})
http_log_req(args, kwargs)
http_log_req(method, uri, args, kwargs)
data = None
if "data" in kwargs:
# The 'data' kwarg is used when you don't want json encoding.
@ -72,27 +73,26 @@ def request(method, uri, *args, **kwargs):
return resp, body
def http_log_req(args, kwargs):
def http_log_req(method, uri, args, kwargs):
"""
When pyrax.get_http_debug() is True, outputs the equivalent `curl`
command for the API request being made.
"""
if not pyrax.get_http_debug():
return
string_parts = ["curl -i"]
string_parts = ["curl -i -X %s" % method]
for element in args:
if element in ("GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"):
string_parts.append(" -X %s" % element)
else:
string_parts.append(" %s" % element)
string_parts.append("%s" % element)
for element in kwargs["headers"]:
header = " -H '%s: %s'" % (element, kwargs["headers"][element])
header = "-H '%s: %s'" % (element, kwargs["headers"][element])
string_parts.append(header)
pyrax._logger.debug("\nREQ: %s\n" % "".join(string_parts))
string_parts.append(uri)
log = logging.getLogger("pyrax")
log.debug("\nREQ: %s\n" % " ".join(string_parts))
if "body" in kwargs:
pyrax._logger.debug("REQ BODY: %s\n" % (kwargs["body"]))
if "data" in kwargs:
pyrax._logger.debug("REQ DATA: %s\n" % (kwargs["data"]))
def http_log_resp(resp, body):
@ -102,4 +102,7 @@ def http_log_resp(resp, body):
"""
if not pyrax.get_http_debug():
return
pyrax._logger.debug("RESP: %s %s\n", resp, body)
log = logging.getLogger("pyrax")
log.debug("RESP: %s\n%s", resp, resp.headers)
if body:
log.debug("RESP BODY: %s", body)

8
awx/lib/site-packages/pyrax/identity/keystone_identity.py
View File

@ -1,12 +1,14 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from __future__ import absolute_import
import pyrax
from pyrax.base_identity import BaseAuth
import pyrax.exceptions as exc
from ..base_identity import BaseIdentity
from .. import exceptions as exc
class KeystoneIdentity(BaseAuth):
class KeystoneIdentity(BaseIdentity):
"""
Implements the Keystone-specific behaviors for Identity. In most
cases you will want to create specific subclasses to implement the

171
awx/lib/site-packages/pyrax/identity/rax_identity.py
View File

@ -1,17 +1,21 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
from six.moves import configparser
from __future__ import absolute_import
from pyrax.base_identity import BaseAuth
from pyrax.base_identity import User
import pyrax.exceptions as exc
import pyrax.utils as utils
from six.moves import configparser as ConfigParser
import pyrax
from ..base_identity import BaseIdentity
from ..base_identity import User
from ..cloudnetworks import CloudNetworkClient
from .. import exceptions as exc
from .. import utils as utils
AUTH_ENDPOINT = "https://identity.api.rackspacecloud.com/v2.0/"
class RaxIdentity(BaseAuth):
class RaxIdentity(BaseIdentity):
"""
This class handles all of the authentication requirements for working
with the Rackspace Cloud.
@ -21,7 +25,8 @@ class RaxIdentity(BaseAuth):
def _get_auth_endpoint(self):
return self._auth_endpoint or AUTH_ENDPOINT
return (self._auth_endpoint or pyrax.get_setting("auth_endpoint")
or AUTH_ENDPOINT)
def _read_credential_file(self, cfg):
@ -31,12 +36,12 @@ class RaxIdentity(BaseAuth):
self.username = cfg.get("rackspace_cloud", "username")
try:
self.password = cfg.get("rackspace_cloud", "api_key", raw=True)
except configparser.NoOptionError as e:
except ConfigParser.NoOptionError as e:
# Allow either the use of either 'api_key' or 'password'.
self.password = cfg.get("rackspace_cloud", "password", raw=True)
def _get_credentials(self):
def _format_credentials(self):
"""
Returns the current credentials in the format expected by the
authentication service. Note that by default Rackspace credentials
@ -47,23 +52,40 @@ class RaxIdentity(BaseAuth):
if self._creds_style == "apikey":
return {"auth": {"RAX-KSKEY:apiKeyCredentials":
{"username": "%s" % self.username,
"apiKey": "%s" % self.password}}}
"apiKey": "%s" % self.api_key}}}
else:
# Return in the default password-style
return super(RaxIdentity, self)._get_credentials()
return super(RaxIdentity, self)._format_credentials()
def authenticate(self):
def set_credentials(self, username, password=None, region=None,
tenant_id=None, authenticate=False):
"""
Sets the username and password directly. Because Rackspace auth uses
the api_key, make sure that any old values are cleared.
"""
self.api_key = None
super(RaxIdentity, self).set_credentials(username, password=password,
region=region, tenant_id=tenant_id, authenticate=authenticate)
def authenticate(self, username=None, password=None, api_key=None,
tenant_id=None, connect=False):
"""
If the user's credentials include an API key, the default behavior will
work. But if they are using a password, the initial attempt will fail,
so try again, but this time using the standard password format.
The 'connect' parameter is retained for backwards compatibility. It no
longer has any effect.
"""
try:
super(RaxIdentity, self).authenticate()
super(RaxIdentity, self).authenticate(username=username,
password=password, api_key=api_key, tenant_id=tenant_id)
except exc.AuthenticationFailed:
self._creds_style = "password"
super(RaxIdentity, self).authenticate()
super(RaxIdentity, self).authenticate(username=username,
password=password, api_key=api_key, tenant_id=tenant_id)
def auth_with_token(self, token, tenant_id=None, tenant_name=None):
@ -80,16 +102,16 @@ class RaxIdentity(BaseAuth):
# object_store endpoints. We can then add these to the initial
# endpoints returned by the primary tenant ID, and then continue with
# the auth process.
main_resp = self._call_token_auth(token, tenant_id, tenant_name)
main_body = main_resp.json()
main_resp, main_body = self._call_token_auth(token, tenant_id,
tenant_name)
# Get the swift tenant ID
roles = main_body["access"]["user"]["roles"]
ostore = [role for role in roles
if role["name"] == "object-store:default"]
if ostore:
ostore_tenant_id = ostore[0]["tenantId"]
ostore_resp = self._call_token_auth(token, ostore_tenant_id, None)
ostore_body = ostore_resp.json()
ostore_resp, ostore_body = self._call_token_auth(token,
ostore_tenant_id, None)
ostore_cat = ostore_body["access"]["serviceCatalog"]
main_cat = main_body["access"]["serviceCatalog"]
main_cat.extend(ostore_cat)
@ -106,13 +128,44 @@ class RaxIdentity(BaseAuth):
self._default_region = defreg
def get_client(self, service, region, public=True, cached=True):
"""
Returns the client object for the specified service and region.
By default the public endpoint is used. If you wish to work with a
services internal endpoints, specify `public=False`.
By default, if a client has already been created for the given service,
region, and public values, that will be returned. To force a new client
to be created, pass 'cached=False'.
"""
client_class = None
# Cloud Networks currently uses nova-networks, so it doesn't appear as
# a separate entry in the service catalog. This hack will allow context
# objects to continue to work with Rackspace Cloud Networks. When the
# Neutron service is implemented, this hack will have to be removed.
if service in ("compute:networks", "networks", "network",
"cloudnetworks", "cloud_networks"):
service = "compute"
client_class = CloudNetworkClient
return super(RaxIdentity, self).get_client(service, region,
public=public, cached=cached, client_class=client_class)
def find_user_by_name(self, name):
"""
Returns a User object by searching for the supplied user name. Returns
None if there is no match for the given name.
"""
uri = "users?name=%s" % name
return self._find_user(uri)
return self.get_user(username=name)
def find_user_by_email(self, email):
"""
Returns a User object by searching for the supplied user's email
address. Returns None if there is no match for the given ID.
"""
return self.get_user(email=email)
def find_user_by_id(self, uid):
@ -120,18 +173,42 @@ class RaxIdentity(BaseAuth):
Returns a User object by searching for the supplied user ID. Returns
None if there is no match for the given ID.
"""
uri = "users/%s" % uid
return self._find_user(uri)
return self.get_user(user_id=uid)
def _find_user(self, uri):
"""Handles the 'find' code for both name and ID searches."""
resp = self.method_get(uri)
if resp.status_code in (403, 404):
return None
jusers = resp.json()
user_info = jusers["user"]
return User(self, user_info)
def get_user(self, user_id=None, username=None, email=None):
"""
Returns the user specified by either ID, username or email.
Since more than user can have the same email address, searching by that
term will return a list of 1 or more User objects. Searching by
username or ID will return a single User.
If a user_id that doesn't belong to the current account is searched
for, a Forbidden exception is raised. When searching by username or
email, a NotFound exception is raised if there is no matching user.
"""
if user_id:
uri = "/users/%s" % user_id
elif username:
uri = "/users?name=%s" % username
elif email:
uri = "/users?email=%s" % email
else:
raise ValueError("You must include one of 'user_id', "
"'username', or 'email' when calling get_user().")
resp, resp_body = self.method_get(uri)
if resp.status_code == 404:
raise exc.NotFound("No such user exists.")
users = resp_body.get("users", [])
if users:
return [User(self, user) for user in users]
else:
user = resp_body.get("user", {})
if user:
return User(self, user)
else:
raise exc.NotFound("No such user exists.")
def update_user(self, user, email=None, username=None,
@ -151,24 +228,26 @@ class RaxIdentity(BaseAuth):
if enabled is not None:
upd["enabled"] = enabled
data = {"user": upd}
resp = self.method_put(uri, data=data)
return User(self, resp.json())
resp, resp_body = self.method_put(uri, data=data)
if resp.status_code in (401, 403, 404):
raise exc.AuthorizationFailure("You are not authorized to update "
"users.")
return User(self, resp_body)
def list_credentials(self, user):
def reset_api_key(self, user=None):
"""
Returns a user's non-password credentials.
"""
user_id = utils.get_id(user)
uri = "users/%s/OS-KSADM/credentials" % user_id
return self.method_get(uri)
Resets the API key for the specified user, or if no user is specified,
for the current user. Returns the newly-created API key.
def get_user_credentials(self, user):
Resetting an API key does not invalidate any authenticated sessions,
nor does it revoke any tokens.
"""
Returns a user's non-password credentials.
"""
user_id = utils.get_id(user)
base_uri = "users/%s/OS-KSADM/credentials/RAX-KSKEY:apiKeyCredentials"
uri = base_uri % user_id
return self.method_get(uri)
if user is None:
user_id = utils.get_id(self)
else:
user_id = utils.get_id(user)
uri = "users/%s/OS-KSADM/credentials/" % user_id
uri += "RAX-KSKEY:apiKeyCredentials/RAX-AUTH/reset"
resp, resp_body = self.method_post(uri)
return resp_body.get("RAX-KSKEY:apiKeyCredentials", {}).get("apiKey")

63
awx/lib/site-packages/pyrax/image.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2014 Rackspace
# Copyright (c)2014 Rackspace US, Inc.
# All Rights Reserved.
#
@ -20,6 +20,7 @@
from functools import wraps
import pyrax
from pyrax.object_storage import StorageObject
from pyrax.client import BaseClient
import pyrax.exceptions as exc
from pyrax.manager import BaseManager
@ -251,6 +252,45 @@ class ImageManager(BaseManager):
return ret
def create(self, name, img_format=None, img_container_format=None,
data=None, container=None, obj=None, metadata=None):
"""
Creates a new image with the specified name. The image data can either
be supplied directly in the 'data' parameter, or it can be an image
stored in the object storage service. In the case of the latter, you
can either supply the container and object names, or simply a
StorageObject reference.
You may specify the image and image container formats; if unspecified,
the default of "vhd" for image format and "bare" for image container
format will be used.
NOTE: This is blocking, and may take a while to complete.
"""
if img_format is None:
img_format = "vhd"
if img_container_format is None:
img_container_format = "bare"
headers = {
"X-Image-Meta-name": name,
"X-Image-Meta-disk_format": img_format,
"X-Image-Meta-container_format": img_container_format,
}
if data:
img_data = data
else:
ident = self.api.identity
region = self.api.region_name
clt = ident.get_client("object_store", region)
if not isinstance(obj, StorageObject):
obj = clt.get_object(container, obj)
img_data = obj.fetch()
uri = "%s/images" % self.uri_base
resp, resp_body = self.api.method_post(uri, headers=headers,
data=img_data)
def update(self, img, value_dict):
"""
Accepts an image reference (object or ID) and dictionary of key/value
@ -294,7 +334,8 @@ class ImageManager(BaseManager):
raise exc.InvalidImageMemberStatus("The status value must be one "
"of 'accepted', 'rejected', or 'pending'. Received: '%s'" %
status)
project_id = pyrax.identity.tenant_id
api = self.api
project_id = api.identity.tenant_id
uri = "/%s/%s/members/%s" % (self.uri_base, img_id, project_id)
body = {"status": status}
try:
@ -385,7 +426,10 @@ class ImageTasksManager(BaseManager):
if cont:
# Verify that it exists. If it doesn't, a NoSuchContainer exception
# will be raised.
pyrax.cloudfiles.get_container(cont)
api = self.api
rgn = api.region_name
cf = api.identity.object_store[rgn].client
cf.get_container(cont)
return super(ImageTasksManager, self).create(name, *args, **kwargs)
@ -518,6 +562,19 @@ class ImageClient(BaseClient):
return self._manager.update(img, value_dict)
def create(self, name, img_format=None, data=None, container=None,
obj=None, metadata=None):
"""
Creates a new image with the specified name. The image data can either
be supplied directly in the 'data' parameter, or it can be an image
stored in the object storage service. In the case of the latter, you
can either supply the container and object names, or simply a
StorageObject reference.
"""
return self._manager.create(name, img_format, data=data,
container=container, obj=obj)
def change_image_name(self, img, newname):
"""
Image name can be changed via the update() method. This is simply a

2
awx/lib/site-packages/pyrax/manager.py
View File

@ -1,7 +1,7 @@
# Copyright 2010 Jacob Kaplan-Moss
# Copyright 2011 OpenStack LLC.
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#

3270
awx/lib/site-packages/pyrax/object_storage.py Normal file
View File

File diff suppressed because it is too large Load Diff

4
awx/lib/site-packages/pyrax/queueing.py
View File

@ -1,7 +1,7 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -21,7 +21,7 @@ from functools import wraps
import json
import os
import re
import urlparse
from six.moves import urllib_parse as urlparse
import pyrax
from pyrax.client import BaseClient

6
awx/lib/site-packages/pyrax/resource.py
View File

@ -1,7 +1,7 @@
# Copyright 2010 Jacob Kaplan-Moss
# Copyright 2011 OpenStack LLC.
# Copyright 2012 Rackspace
# Copyright (c)2012 Rackspace US, Inc.
# All Rights Reserved.
#
@ -67,9 +67,11 @@ class BaseResource(object):
Takes the dict returned by the API call and sets the
corresponding attributes on the object.
"""
for (key, val) in info.iteritems():
for (key, val) in six.iteritems(info):
if isinstance(key, six.text_type):
key = key.encode(pyrax.get_encoding())
elif isinstance(key, bytes):
key = key.decode("utf-8")
setattr(self, key, val)

109
awx/lib/site-packages/pyrax/utils.py
View File

@ -111,6 +111,111 @@ class SelfDeletingTempDirectory(object):
shutil.rmtree(self.name)
class DotDict(dict):
"""
Dictionary subclass that allows accessing keys via dot notation.
If the key is not present, an AttributeError is raised.
"""
_att_mapper = {}
_fail = object()
def __init__(self, *args, **kwargs):
super(DotDict, self).__init__(*args, **kwargs)
def __getattr__(self, att):
att = self._att_mapper.get(att, att)
ret = self.get(att, self._fail)
if ret is self._fail:
raise AttributeError("'%s' object has no attribute '%s'" %
(self.__class__.__name__, att))
return ret
__setattr__ = dict.__setitem__
__delattr__ = dict.__delitem__
class ResultsIterator(object):
"""
This object will iterate over all the results for a given type of listing,
no matter how many items exist.
This is an abstract class; subclasses must define the _init_methods()
method to specify what manager method should be called to get the next
batch of results. Both the equivalent of 'list()' and '_list()' may be
specified You may also specify any extra args to be sent when this method
is called.
By default the marker will be unspecified, and the limit will be 1000. You
can override either by specifying them during instantiation.
The 'kwargs' will be converted to attributes. E.g., in this call:
rit = ResultsIterator(mgr, foo="bar")
will result in the object having a 'foo' attribute with the value of 'bar'.
"""
def __init__(self, manager, marker=None, limit=1000, **kwargs):
self.manager = manager
self.marker = marker
self.limit = limit
for att, val in list(kwargs.items()):
setattr(self, att, val)
self.results = []
self.list_method = None
self._list_method = None
self.marker_att = "id"
self.extra_args = tuple()
self._init_methods()
self.next_uri = ""
def _init_methods(self):
"""
Must be implemented in subclasses. For results that return a URI for
the next batch of results, the lower-level '_list_method' will be
called, using that URI. Otherwise, the 'list_method' will be called,
with the paging info from the prior call.
If your class uses an attribute other than 'id' as the marker, set this
object's 'marker_att' to that attribute.
"""
raise NotImplementedError()
def __iter__(self):
return self
def next(self):
"""
Return the next available item. If there are no more items in the
local 'results' list, check if there is a 'next_uri' value. If so,
use that to get the next page of results from the API, and return
the first item from that query.
"""
try:
return self.results.pop(0)
except IndexError:
if self.next_uri is None:
raise StopIteration()
else:
if not self.next_uri:
self.results = self.list_method(marker=self.marker,
limit=self.limit, prefix=self.prefix)
else:
args = self.extra_args
self.results = self._list_method(self.next_uri, *args)
if self.results:
last_res = self.results[-1]
self.marker = getattr(last_res, self.marker_att)
# We should have more results.
try:
return self.results.pop(0)
except IndexError:
raise StopIteration()
def get_checksum(content, encoding="utf8", block_size=8192):
"""
Returns the MD5 checksum in hex for the given content. If 'content'
@ -161,7 +266,7 @@ def _join_chars(chars, length):
"""
Used by the random character functions.
"""
mult = (length / len(chars)) + 1
mult = int(length / len(chars)) + 1
mult_chars = chars * mult
return "".join(random.sample(mult_chars, length))
@ -174,7 +279,7 @@ def random_unicode(length=20):
up to code point 1000.
"""
def get_char():
return unichr(random.randint(32, 1000))
return six.unichr(random.randint(32, 1000))
chars = u"".join([get_char() for ii in six.moves.range(length)])
return _join_chars(chars, length)

2
awx/lib/site-packages/pyrax/version.py
View File

@ -1,4 +1,4 @@
#!/usr/bin/env python
# -*- coding: utf-8 -*-
version = "1.7.2"
version = "1.9.0"