Include local versions of third-party dependencies, particularly those unavailable or outdated as OS packages.

awx/lib/site-packages/celery/app/__init__.py

@@ -0,0 +1,136 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app
+    ~~~~~~~~~~
+
+    Celery Application.
+
+"""
+from __future__ import absolute_import
+from __future__ import with_statement
+
+import os
+
+from celery.local import Proxy
+from celery import _state
+from celery._state import (  # noqa
+    set_default_app,
+    get_current_app as current_app,
+    get_current_task as current_task,
+    _get_active_apps,
+)
+from celery.utils import gen_task_name
+
+from .builtins import shared_task as _shared_task
+from .base import Celery, AppPickler  # noqa
+
+#: Proxy always returning the app set as default.
+default_app = Proxy(lambda: _state.default_app)
+
+#: Function returning the app provided or the default app if none.
+#:
+#: The environment variable :envvar:`CELERY_TRACE_APP` is used to
+#: trace app leaks.  When enabled an exception is raised if there
+#: is no active app.
+app_or_default = None
+
+#: The 'default' loader is the default loader used by old applications.
+#: This is deprecated and should no longer be used as it's set too early
+#: to be affected by --loader argument.
+default_loader = os.environ.get('CELERY_LOADER') or 'default'  # XXX
+
+
+def bugreport():
+    return current_app().bugreport()
+
+
+def _app_or_default(app=None):
+    if app is None:
+        return _state.get_current_app()
+    return app
+
+
+def _app_or_default_trace(app=None):  # pragma: no cover
+    from traceback import print_stack
+    from billiard import current_process
+    if app is None:
+        if getattr(_state._tls, 'current_app', None):
+            print('-- RETURNING TO CURRENT APP --')  # noqa+
+            print_stack()
+            return _state._tls.current_app
+        if current_process()._name == 'MainProcess':
+            raise Exception('DEFAULT APP')
+        print('-- RETURNING TO DEFAULT APP --')      # noqa+
+        print_stack()
+        return _state.default_app
+    return app
+
+
+def enable_trace():
+    global app_or_default
+    app_or_default = _app_or_default_trace
+
+
+def disable_trace():
+    global app_or_default
+    app_or_default = _app_or_default
+
+if os.environ.get('CELERY_TRACE_APP'):  # pragma: no cover
+    enable_trace()
+else:
+    disable_trace()
+
+App = Celery  # XXX Compat
+
+
+def shared_task(*args, **kwargs):
+    """Task decorator that creates shared tasks,
+    and returns a proxy that always returns the task from the current apps
+    task registry.
+
+    This can be used by library authors to create tasks that will work
+    for any app environment.
+
+    Example:
+
+        >>> from celery import Celery, shared_task
+        >>> @shared_task
+        ... def add(x, y):
+        ...     return x + y
+
+        >>> app1 = Celery(broker='amqp://')
+        >>> add.app is app1
+        True
+
+        >>> app2 = Celery(broker='redis://')
+        >>> add.app is app2
+
+    """
+
+    def create_shared_task(**options):
+
+        def __inner(fun):
+            name = options.get('name')
+            # Set as shared task so that unfinalized apps,
+            # and future apps will load the task.
+            _shared_task(lambda app: app._task_from_fun(fun, **options))
+
+            # Force all finalized apps to take this task as well.
+            for app in _get_active_apps():
+                if app.finalized:
+                    with app._finalize_mutex:
+                        app._task_from_fun(fun, **options)
+
+            # Returns a proxy that always gets the task from the current
+            # apps task registry.
+            def task_by_cons():
+                app = current_app()
+                return app.tasks[
+                    name or gen_task_name(app, fun.__name__, fun.__module__)
+                ]
+            return Proxy(task_by_cons)
+        return __inner
+
+    if len(args) == 1 and callable(args[0]):
+        return create_shared_task(**kwargs)(args[0])
+    return create_shared_task(*args, **kwargs)

awx/lib/site-packages/celery/app/abstract.py

@@ -0,0 +1,63 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.abstract
+    ~~~~~~~~~~~~~~~~~~~
+
+    Abstract class that takes default attribute values
+    from the configuration.
+
+"""
+from __future__ import absolute_import
+
+
+class from_config(object):
+
+    def __init__(self, key=None):
+        self.key = key
+
+    def get_key(self, attr):
+        return attr if self.key is None else self.key
+
+
+class _configurated(type):
+
+    def __new__(cls, name, bases, attrs):
+        attrs['__confopts__'] = dict((attr, spec.get_key(attr))
+                                     for attr, spec in attrs.iteritems()
+                                     if isinstance(spec, from_config))
+        inherit_from = attrs.get('inherit_confopts', ())
+        for subcls in bases:
+            try:
+                attrs['__confopts__'].update(subcls.__confopts__)
+            except AttributeError:
+                pass
+        for subcls in inherit_from:
+            attrs['__confopts__'].update(subcls.__confopts__)
+        attrs = dict((k, v if not isinstance(v, from_config) else None)
+                     for k, v in attrs.iteritems())
+        return super(_configurated, cls).__new__(cls, name, bases, attrs)
+
+
+class configurated(object):
+    __metaclass__ = _configurated
+
+    def setup_defaults(self, kwargs, namespace='celery'):
+        confopts = self.__confopts__
+        app, find = self.app, self.app.conf.find_value_for_key
+
+        for attr, keyname in confopts.iteritems():
+            try:
+                value = kwargs[attr]
+            except KeyError:
+                value = find(keyname, namespace)
+            else:
+                if value is None:
+                    value = find(keyname, namespace)
+            setattr(self, attr, value)
+
+        for attr_name, attr_value in kwargs.iteritems():
+            if attr_name not in confopts and attr_value is not None:
+                setattr(self, attr_name, attr_value)
+
+    def confopts_as_dict(self):
+        return dict((key, getattr(self, key)) for key in self.__confopts__)

awx/lib/site-packages/celery/app/amqp.py

@@ -0,0 +1,425 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.amqp
+    ~~~~~~~~~~~~~~~
+
+    Sending and receiving messages using Kombu.
+
+"""
+from __future__ import absolute_import
+
+from datetime import timedelta
+from weakref import WeakValueDictionary
+
+from kombu import Connection, Consumer, Exchange, Producer, Queue
+from kombu.common import entry_to_queue
+from kombu.pools import ProducerPool
+from kombu.utils import cached_property, uuid
+from kombu.utils.encoding import safe_repr
+
+from celery import signals
+from celery.utils.text import indent as textindent
+
+from . import app_or_default
+from . import routes as _routes
+
+#: Human readable queue declaration.
+QUEUE_FORMAT = """
+.> %(name)s exchange:%(exchange)s(%(exchange_type)s) binding:%(routing_key)s
+"""
+
+
+class Queues(dict):
+    """Queue name⇒ declaration mapping.
+
+    :param queues: Initial list/tuple or dict of queues.
+    :keyword create_missing: By default any unknown queues will be
+                             added automatically, but if disabled
+                             the occurrence of unknown queues
+                             in `wanted` will raise :exc:`KeyError`.
+    :keyword ha_policy: Default HA policy for queues with none set.
+
+
+    """
+    #: If set, this is a subset of queues to consume from.
+    #: The rest of the queues are then used for routing only.
+    _consume_from = None
+
+    def __init__(self, queues=None, default_exchange=None,
+                 create_missing=True, ha_policy=None):
+        dict.__init__(self)
+        self.aliases = WeakValueDictionary()
+        self.default_exchange = default_exchange
+        self.create_missing = create_missing
+        self.ha_policy = ha_policy
+        if isinstance(queues, (tuple, list)):
+            queues = dict((q.name, q) for q in queues)
+        for name, q in (queues or {}).iteritems():
+            self.add(q) if isinstance(q, Queue) else self.add_compat(name, **q)
+
+    def __getitem__(self, name):
+        try:
+            return self.aliases[name]
+        except KeyError:
+            return dict.__getitem__(self, name)
+
+    def __setitem__(self, name, queue):
+        if self.default_exchange and (not queue.exchange or
+                                      not queue.exchange.name):
+            queue.exchange = self.default_exchange
+        dict.__setitem__(self, name, queue)
+        if queue.alias:
+            self.aliases[queue.alias] = queue
+
+    def __missing__(self, name):
+        if self.create_missing:
+            return self.add(self.new_missing(name))
+        raise KeyError(name)
+
+    def add(self, queue, **kwargs):
+        """Add new queue.
+
+        :param queue: Name of the queue.
+        :keyword exchange: Name of the exchange.
+        :keyword routing_key: Binding key.
+        :keyword exchange_type: Type of exchange.
+        :keyword \*\*options: Additional declaration options.
+
+        """
+        if not isinstance(queue, Queue):
+            return self.add_compat(queue, **kwargs)
+        if self.ha_policy:
+            if queue.queue_arguments is None:
+                queue.queue_arguments = {}
+            self._set_ha_policy(queue.queue_arguments)
+        self[queue.name] = queue
+        return queue
+
+    def add_compat(self, name, **options):
+        # docs used to use binding_key as routing key
+        options.setdefault('routing_key', options.get('binding_key'))
+        if options['routing_key'] is None:
+            options['routing_key'] = name
+        if self.ha_policy is not None:
+            self._set_ha_policy(options.setdefault('queue_arguments', {}))
+        q = self[name] = entry_to_queue(name, **options)
+        return q
+
+    def _set_ha_policy(self, args):
+        policy = self.ha_policy
+        if isinstance(policy, (list, tuple)):
+            return args.update({'x-ha-policy': 'nodes',
+                                'x-ha-policy-params': list(policy)})
+        args['x-ha-policy'] = policy
+
+    def format(self, indent=0, indent_first=True):
+        """Format routing table into string for log dumps."""
+        active = self.consume_from
+        if not active:
+            return ''
+        info = [
+            QUEUE_FORMAT.strip() % {
+                'name': (name + ':').ljust(12),
+                'exchange': q.exchange.name,
+                'exchange_type': q.exchange.type,
+                'routing_key': q.routing_key}
+            for name, q in sorted(active.iteritems())]
+        if indent_first:
+            return textindent('\n'.join(info), indent)
+        return info[0] + '\n' + textindent('\n'.join(info[1:]), indent)
+
+    def select_add(self, queue, **kwargs):
+        """Add new task queue that will be consumed from even when
+        a subset has been selected using the :option:`-Q` option."""
+        q = self.add(queue, **kwargs)
+        if self._consume_from is not None:
+            self._consume_from[q.name] = q
+        return q
+
+    def select_subset(self, wanted):
+        """Sets :attr:`consume_from` by selecting a subset of the
+        currently defined queues.
+
+        :param wanted: List of wanted queue names.
+        """
+        if wanted:
+            self._consume_from = dict((name, self[name]) for name in wanted)
+
+    def select_remove(self, queue):
+        if self._consume_from is None:
+            self.select_subset(k for k in self if k != queue)
+        else:
+            self._consume_from.pop(queue, None)
+
+    def new_missing(self, name):
+        return Queue(name, Exchange(name), name)
+
+    @property
+    def consume_from(self):
+        if self._consume_from is not None:
+            return self._consume_from
+        return self
+
+
+class TaskProducer(Producer):
+    app = None
+    auto_declare = False
+    retry = False
+    retry_policy = None
+    utc = True
+    event_dispatcher = None
+    send_sent_event = False
+
+    def __init__(self, channel=None, exchange=None, *args, **kwargs):
+        self.retry = kwargs.pop('retry', self.retry)
+        self.retry_policy = kwargs.pop('retry_policy',
+                                       self.retry_policy or {})
+        self.send_sent_event = kwargs.pop('send_sent_event',
+                                          self.send_sent_event)
+        exchange = exchange or self.exchange
+        self.queues = self.app.amqp.queues  # shortcut
+        self.default_queue = self.app.amqp.default_queue
+        super(TaskProducer, self).__init__(channel, exchange, *args, **kwargs)
+
+    def publish_task(self, task_name, task_args=None, task_kwargs=None,
+                     countdown=None, eta=None, task_id=None, group_id=None,
+                     taskset_id=None,  # compat alias to group_id
+                     expires=None, exchange=None, exchange_type=None,
+                     event_dispatcher=None, retry=None, retry_policy=None,
+                     queue=None, now=None, retries=0, chord=None,
+                     callbacks=None, errbacks=None, routing_key=None,
+                     serializer=None, delivery_mode=None, compression=None,
+                     declare=None, **kwargs):
+        """Send task message."""
+
+        qname = queue
+        if queue is None and exchange is None:
+            queue = self.default_queue
+        if queue is not None:
+            if isinstance(queue, basestring):
+                qname, queue = queue, self.queues[queue]
+            else:
+                qname = queue.name
+            exchange = exchange or queue.exchange.name
+            routing_key = routing_key or queue.routing_key
+        declare = declare or ([queue] if queue else [])
+
+        # merge default and custom policy
+        retry = self.retry if retry is None else retry
+        _rp = (dict(self.retry_policy, **retry_policy) if retry_policy
+               else self.retry_policy)
+        task_id = task_id or uuid()
+        task_args = task_args or []
+        task_kwargs = task_kwargs or {}
+        if not isinstance(task_args, (list, tuple)):
+            raise ValueError('task args must be a list or tuple')
+        if not isinstance(task_kwargs, dict):
+            raise ValueError('task kwargs must be a dictionary')
+        if countdown:  # Convert countdown to ETA.
+            now = now or self.app.now()
+            eta = now + timedelta(seconds=countdown)
+        if isinstance(expires, (int, float)):
+            now = now or self.app.now()
+            expires = now + timedelta(seconds=expires)
+        eta = eta and eta.isoformat()
+        expires = expires and expires.isoformat()
+
+        body = {
+            'task': task_name,
+            'id': task_id,
+            'args': task_args,
+            'kwargs': task_kwargs,
+            'retries': retries or 0,
+            'eta': eta,
+            'expires': expires,
+            'utc': self.utc,
+            'callbacks': callbacks,
+            'errbacks': errbacks,
+            'taskset': group_id or taskset_id,
+            'chord': chord,
+        }
+
+        self.publish(
+            body,
+            exchange=exchange, routing_key=routing_key,
+            serializer=serializer or self.serializer,
+            compression=compression or self.compression,
+            retry=retry, retry_policy=_rp,
+            delivery_mode=delivery_mode, declare=declare,
+            **kwargs
+        )
+
+        signals.task_sent.send(sender=task_name, **body)
+        if self.send_sent_event:
+            evd = event_dispatcher or self.event_dispatcher
+            exname = exchange or self.exchange
+            if isinstance(exname, Exchange):
+                exname = exname.name
+            evd.publish(
+                'task-sent',
+                {
+                    'uuid': task_id,
+                    'name': task_name,
+                    'args': safe_repr(task_args),
+                    'kwargs': safe_repr(task_kwargs),
+                    'retries': retries,
+                    'eta': eta,
+                    'expires': expires,
+                    'queue': qname,
+                    'exchange': exname,
+                    'routing_key': routing_key,
+                },
+                self, retry=retry, retry_policy=retry_policy,
+            )
+        return task_id
+    delay_task = publish_task   # XXX Compat
+
+    @cached_property
+    def event_dispatcher(self):
+        # We call Dispatcher.publish with a custom producer
+        # so don't need the dispatcher to be "enabled".
+        return self.app.events.Dispatcher(enabled=False)
+
+
+class TaskPublisher(TaskProducer):
+    """Deprecated version of :class:`TaskProducer`."""
+
+    def __init__(self, channel=None, exchange=None, *args, **kwargs):
+        self.app = app_or_default(kwargs.pop('app', self.app))
+        self.retry = kwargs.pop('retry', self.retry)
+        self.retry_policy = kwargs.pop('retry_policy',
+                                       self.retry_policy or {})
+        exchange = exchange or self.exchange
+        if not isinstance(exchange, Exchange):
+            exchange = Exchange(exchange,
+                                kwargs.pop('exchange_type', 'direct'))
+        self.queues = self.app.amqp.queues  # shortcut
+        super(TaskPublisher, self).__init__(channel, exchange, *args, **kwargs)
+
+
+class TaskConsumer(Consumer):
+    app = None
+
+    def __init__(self, channel, queues=None, app=None, accept=None, **kw):
+        self.app = app or self.app
+        if accept is None:
+            accept = self.app.conf.CELERY_ACCEPT_CONTENT
+        super(TaskConsumer, self).__init__(
+            channel,
+            queues or self.app.amqp.queues.consume_from.values(),
+            accept=accept,
+            **kw
+        )
+
+
+class AMQP(object):
+    Connection = Connection
+    Consumer = Consumer
+
+    #: compat alias to Connection
+    BrokerConnection = Connection
+
+    producer_cls = TaskProducer
+    consumer_cls = TaskConsumer
+
+    #: Cached and prepared routing table.
+    _rtable = None
+
+    #: Underlying producer pool instance automatically
+    #: set by the :attr:`producer_pool`.
+    _producer_pool = None
+
+    def __init__(self, app):
+        self.app = app
+
+    def flush_routes(self):
+        self._rtable = _routes.prepare(self.app.conf.CELERY_ROUTES)
+
+    def Queues(self, queues, create_missing=None, ha_policy=None):
+        """Create new :class:`Queues` instance, using queue defaults
+        from the current configuration."""
+        conf = self.app.conf
+        if create_missing is None:
+            create_missing = conf.CELERY_CREATE_MISSING_QUEUES
+        if ha_policy is None:
+            ha_policy = conf.CELERY_QUEUE_HA_POLICY
+        if not queues and conf.CELERY_DEFAULT_QUEUE:
+            queues = (Queue(conf.CELERY_DEFAULT_QUEUE,
+                            exchange=self.default_exchange,
+                            routing_key=conf.CELERY_DEFAULT_ROUTING_KEY), )
+        return Queues(queues, self.default_exchange, create_missing, ha_policy)
+
+    def Router(self, queues=None, create_missing=None):
+        """Returns the current task router."""
+        return _routes.Router(self.routes, queues or self.queues,
+                              self.app.either('CELERY_CREATE_MISSING_QUEUES',
+                                              create_missing), app=self.app)
+
+    @cached_property
+    def TaskConsumer(self):
+        """Return consumer configured to consume from the queues
+        we are configured for (``app.amqp.queues.consume_from``)."""
+        return self.app.subclass_with_self(self.consumer_cls,
+                                           reverse='amqp.TaskConsumer')
+    get_task_consumer = TaskConsumer  # XXX compat
+
+    @cached_property
+    def TaskProducer(self):
+        """Returns publisher used to send tasks.
+
+        You should use `app.send_task` instead.
+
+        """
+        conf = self.app.conf
+        return self.app.subclass_with_self(
+            self.producer_cls,
+            reverse='amqp.TaskProducer',
+            exchange=self.default_exchange,
+            routing_key=conf.CELERY_DEFAULT_ROUTING_KEY,
+            serializer=conf.CELERY_TASK_SERIALIZER,
+            compression=conf.CELERY_MESSAGE_COMPRESSION,
+            retry=conf.CELERY_TASK_PUBLISH_RETRY,
+            retry_policy=conf.CELERY_TASK_PUBLISH_RETRY_POLICY,
+            send_sent_event=conf.CELERY_SEND_TASK_SENT_EVENT,
+            utc=conf.CELERY_ENABLE_UTC,
+        )
+    TaskPublisher = TaskProducer  # compat
+
+    @cached_property
+    def default_queue(self):
+        return self.queues[self.app.conf.CELERY_DEFAULT_QUEUE]
+
+    @cached_property
+    def queues(self):
+        """Queue name⇒ declaration mapping."""
+        return self.Queues(self.app.conf.CELERY_QUEUES)
+
+    @queues.setter  # noqa
+    def queues(self, queues):
+        return self.Queues(queues)
+
+    @property
+    def routes(self):
+        if self._rtable is None:
+            self.flush_routes()
+        return self._rtable
+
+    @cached_property
+    def router(self):
+        return self.Router()
+
+    @property
+    def producer_pool(self):
+        if self._producer_pool is None:
+            self._producer_pool = ProducerPool(
+                self.app.pool,
+                limit=self.app.pool.limit,
+                Producer=self.TaskProducer,
+            )
+        return self._producer_pool
+    publisher_pool = producer_pool  # compat alias
+
+    @cached_property
+    def default_exchange(self):
+        return Exchange(self.app.conf.CELERY_DEFAULT_EXCHANGE,
+                        self.app.conf.CELERY_DEFAULT_EXCHANGE_TYPE)

awx/lib/site-packages/celery/app/annotations.py

@@ -0,0 +1,55 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.annotations
+    ~~~~~~~~~~~~~~~~~~~~~~
+
+    Annotations is a nice term for moneky patching
+    task classes in the configuration.
+
+    This prepares and performs the annotations in the
+    :setting:`CELERY_ANNOTATIONS` setting.
+
+"""
+from __future__ import absolute_import
+
+from celery.utils.functional import firstmethod, mpromise
+from celery.utils.imports import instantiate
+
+_first_match = firstmethod('annotate')
+_first_match_any = firstmethod('annotate_any')
+
+
+def resolve_all(anno, task):
+    return (r for r in (_first_match(anno, task), _first_match_any(anno)) if r)
+
+
+class MapAnnotation(dict):
+
+    def annotate_any(self):
+        try:
+            return dict(self['*'])
+        except KeyError:
+            pass
+
+    def annotate(self, task):
+        try:
+            return dict(self[task.name])
+        except KeyError:
+            pass
+
+
+def prepare(annotations):
+    """Expands the :setting:`CELERY_ANNOTATIONS` setting."""
+
+    def expand_annotation(annotation):
+        if isinstance(annotation, dict):
+            return MapAnnotation(annotation)
+        elif isinstance(annotation, basestring):
+            return mpromise(instantiate, annotation)
+        return annotation
+
+    if annotations is None:
+        return ()
+    elif not isinstance(annotations, (list, tuple)):
+        annotations = (annotations, )
+    return [expand_annotation(anno) for anno in annotations]

awx/lib/site-packages/celery/app/base.py

@@ -0,0 +1,516 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.base
+    ~~~~~~~~~~~~~~~
+
+    Actual App instance implementation.
+
+"""
+from __future__ import absolute_import
+from __future__ import with_statement
+
+import os
+import threading
+import warnings
+
+from collections import deque
+from contextlib import contextmanager
+from copy import deepcopy
+from functools import wraps
+
+from billiard.util import register_after_fork
+from kombu.clocks import LamportClock
+from kombu.utils import cached_property
+
+from celery import platforms
+from celery.exceptions import AlwaysEagerIgnored, ImproperlyConfigured
+from celery.loaders import get_loader_cls
+from celery.local import PromiseProxy, maybe_evaluate
+from celery._state import _task_stack, _tls, get_current_app, _register_app
+from celery.utils.functional import first
+from celery.utils.imports import instantiate, symbol_by_name
+
+from .annotations import prepare as prepare_annotations
+from .builtins import shared_task, load_shared_tasks
+from .defaults import DEFAULTS, find_deprecated_settings
+from .registry import TaskRegistry
+from .utils import AppPickler, Settings, bugreport, _unpickle_app
+
+_EXECV = os.environ.get('FORKED_BY_MULTIPROCESSING')
+
+
+def _unpickle_appattr(reverse_name, args):
+    """Given an attribute name and a list of args, gets
+    the attribute from the current app and calls it."""
+    return get_current_app()._rgetattr(reverse_name)(*args)
+
+
+class Celery(object):
+    Pickler = AppPickler
+
+    SYSTEM = platforms.SYSTEM
+    IS_OSX, IS_WINDOWS = platforms.IS_OSX, platforms.IS_WINDOWS
+
+    amqp_cls = 'celery.app.amqp:AMQP'
+    backend_cls = None
+    events_cls = 'celery.events:Events'
+    loader_cls = 'celery.loaders.app:AppLoader'
+    log_cls = 'celery.app.log:Logging'
+    control_cls = 'celery.app.control:Control'
+    registry_cls = TaskRegistry
+    _pool = None
+
+    def __init__(self, main=None, loader=None, backend=None,
+                 amqp=None, events=None, log=None, control=None,
+                 set_as_current=True, accept_magic_kwargs=False,
+                 tasks=None, broker=None, include=None, changes=None,
+                 config_source=None,
+                 **kwargs):
+        self.clock = LamportClock()
+        self.main = main
+        self.amqp_cls = amqp or self.amqp_cls
+        self.backend_cls = backend or self.backend_cls
+        self.events_cls = events or self.events_cls
+        self.loader_cls = loader or self.loader_cls
+        self.log_cls = log or self.log_cls
+        self.control_cls = control or self.control_cls
+        self.set_as_current = set_as_current
+        self.registry_cls = symbol_by_name(self.registry_cls)
+        self.accept_magic_kwargs = accept_magic_kwargs
+        self._config_source = config_source
+
+        self.configured = False
+        self._pending_defaults = deque()
+
+        self.finalized = False
+        self._finalize_mutex = threading.Lock()
+        self._pending = deque()
+        self._tasks = tasks
+        if not isinstance(self._tasks, TaskRegistry):
+            self._tasks = TaskRegistry(self._tasks or {})
+
+        # these options are moved to the config to
+        # simplify pickling of the app object.
+        self._preconf = changes or {}
+        if broker:
+            self._preconf['BROKER_URL'] = broker
+        if include:
+            self._preconf['CELERY_IMPORTS'] = include
+
+        if self.set_as_current:
+            self.set_current()
+
+        # See Issue #1126
+        # this is used when pickling the app object so that configuration
+        # is reread without having to pickle the contents
+        # (which is often unpickleable anyway)
+        if self._config_source:
+            self.config_from_object(self._config_source)
+
+        self.on_init()
+        _register_app(self)
+
+    def set_current(self):
+        _tls.current_app = self
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *exc_info):
+        self.close()
+
+    def close(self):
+        self._maybe_close_pool()
+
+    def on_init(self):
+        """Optional callback called at init."""
+        pass
+
+    def start(self, argv=None):
+        return instantiate(
+            'celery.bin.celery:CeleryCommand',
+            app=self).execute_from_commandline(argv)
+
+    def worker_main(self, argv=None):
+        return instantiate(
+            'celery.bin.celeryd:WorkerCommand',
+            app=self).execute_from_commandline(argv)
+
+    def task(self, *args, **opts):
+        """Creates new task class from any callable."""
+        if _EXECV and not opts.get('_force_evaluate'):
+            # When using execv the task in the original module will point to a
+            # different app, so doing things like 'add.request' will point to
+            # a differnt task instance.  This makes sure it will always use
+            # the task instance from the current app.
+            # Really need a better solution for this :(
+            from . import shared_task as proxies_to_curapp
+            opts['_force_evaluate'] = True  # XXX Py2.5
+            return proxies_to_curapp(*args, **opts)
+
+        def inner_create_task_cls(shared=True, filter=None, **opts):
+            _filt = filter  # stupid 2to3
+
+            def _create_task_cls(fun):
+                if shared:
+                    cons = lambda app: app._task_from_fun(fun, **opts)
+                    cons.__name__ = fun.__name__
+                    shared_task(cons)
+                if self.accept_magic_kwargs:  # compat mode
+                    task = self._task_from_fun(fun, **opts)
+                    if filter:
+                        task = filter(task)
+                    return task
+
+                # return a proxy object that is only evaluated when first used
+                promise = PromiseProxy(self._task_from_fun, (fun, ), opts)
+                self._pending.append(promise)
+                if _filt:
+                    return _filt(promise)
+                return promise
+
+            return _create_task_cls
+
+        if len(args) == 1 and callable(args[0]):
+            return inner_create_task_cls(**opts)(*args)
+        if args:
+            raise TypeError(
+                'task() takes no arguments (%s given)' % (len(args, )))
+        return inner_create_task_cls(**opts)
+
+    def _task_from_fun(self, fun, **options):
+        base = options.pop('base', None) or self.Task
+
+        T = type(fun.__name__, (base, ), dict({
+            'app': self,
+            'accept_magic_kwargs': False,
+            'run': staticmethod(fun),
+            '__doc__': fun.__doc__,
+            '__module__': fun.__module__}, **options))()
+        task = self._tasks[T.name]  # return global instance.
+        task.bind(self)
+        return task
+
+    def finalize(self):
+        with self._finalize_mutex:
+            if not self.finalized:
+                self.finalized = True
+                load_shared_tasks(self)
+
+                pending = self._pending
+                while pending:
+                    maybe_evaluate(pending.popleft())
+
+                for task in self._tasks.itervalues():
+                    task.bind(self)
+
+    def add_defaults(self, fun):
+        if not callable(fun):
+            d, fun = fun, lambda: d
+        if self.configured:
+            return self.conf.add_defaults(fun())
+        self._pending_defaults.append(fun)
+
+    def config_from_object(self, obj, silent=False):
+        del(self.conf)
+        self._config_source = obj
+        return self.loader.config_from_object(obj, silent=silent)
+
+    def config_from_envvar(self, variable_name, silent=False):
+        module_name = os.environ.get(variable_name)
+        if not module_name:
+            if silent:
+                return False
+            raise ImproperlyConfigured(self.error_envvar_not_set % module_name)
+        return self.config_from_object(module_name, silent=silent)
+
+    def config_from_cmdline(self, argv, namespace='celery'):
+        self.conf.update(self.loader.cmdline_config_parser(argv, namespace))
+
+    def send_task(self, name, args=None, kwargs=None, countdown=None,
+                  eta=None, task_id=None, producer=None, connection=None,
+                  result_cls=None, expires=None, queues=None, publisher=None,
+                  **options):
+        producer = producer or publisher  # XXX compat
+        if self.conf.CELERY_ALWAYS_EAGER:  # pragma: no cover
+            warnings.warn(AlwaysEagerIgnored(
+                'CELERY_ALWAYS_EAGER has no effect on send_task'))
+
+        result_cls = result_cls or self.AsyncResult
+        router = self.amqp.Router(queues)
+        options.setdefault('compression',
+                           self.conf.CELERY_MESSAGE_COMPRESSION)
+        options = router.route(options, name, args, kwargs)
+        with self.producer_or_acquire(producer) as producer:
+            return result_cls(producer.publish_task(
+                name, args, kwargs,
+                task_id=task_id,
+                countdown=countdown, eta=eta,
+                expires=expires, **options
+            ))
+
+    def connection(self, hostname=None, userid=None,
+                   password=None, virtual_host=None, port=None, ssl=None,
+                   insist=None, connect_timeout=None, transport=None,
+                   transport_options=None, heartbeat=None, **kwargs):
+        conf = self.conf
+        return self.amqp.Connection(
+            hostname or conf.BROKER_HOST,
+            userid or conf.BROKER_USER,
+            password or conf.BROKER_PASSWORD,
+            virtual_host or conf.BROKER_VHOST,
+            port or conf.BROKER_PORT,
+            transport=transport or conf.BROKER_TRANSPORT,
+            insist=self.either('BROKER_INSIST', insist),
+            ssl=self.either('BROKER_USE_SSL', ssl),
+            connect_timeout=self.either(
+                'BROKER_CONNECTION_TIMEOUT', connect_timeout),
+            heartbeat=heartbeat,
+            transport_options=dict(conf.BROKER_TRANSPORT_OPTIONS,
+                                   **transport_options or {}))
+    broker_connection = connection
+
+    @contextmanager
+    def connection_or_acquire(self, connection=None, pool=True,
+                              *args, **kwargs):
+        if connection:
+            yield connection
+        else:
+            if pool:
+                with self.pool.acquire(block=True) as connection:
+                    yield connection
+            else:
+                with self.connection() as connection:
+                    yield connection
+    default_connection = connection_or_acquire  # XXX compat
+
+    @contextmanager
+    def producer_or_acquire(self, producer=None):
+        if producer:
+            yield producer
+        else:
+            with self.amqp.producer_pool.acquire(block=True) as producer:
+                yield producer
+    default_producer = producer_or_acquire  # XXX compat
+
+    def with_default_connection(self, fun):
+        """With any function accepting a `connection`
+        keyword argument, establishes a default connection if one is
+        not already passed to it.
+
+        Any automatically established connection will be closed after
+        the function returns.
+
+        **Deprecated**
+
+        Use ``with app.connection_or_acquire(connection)`` instead.
+
+        """
+        @wraps(fun)
+        def _inner(*args, **kwargs):
+            connection = kwargs.pop('connection', None)
+            with self.connection_or_acquire(connection) as c:
+                return fun(*args, **dict(kwargs, connection=c))
+        return _inner
+
+    def prepare_config(self, c):
+        """Prepare configuration before it is merged with the defaults."""
+        return find_deprecated_settings(c)
+
+    def now(self):
+        return self.loader.now(utc=self.conf.CELERY_ENABLE_UTC)
+
+    def mail_admins(self, subject, body, fail_silently=False):
+        if self.conf.ADMINS:
+            to = [admin_email for _, admin_email in self.conf.ADMINS]
+            return self.loader.mail_admins(
+                subject, body, fail_silently, to=to,
+                sender=self.conf.SERVER_EMAIL,
+                host=self.conf.EMAIL_HOST,
+                port=self.conf.EMAIL_PORT,
+                user=self.conf.EMAIL_HOST_USER,
+                password=self.conf.EMAIL_HOST_PASSWORD,
+                timeout=self.conf.EMAIL_TIMEOUT,
+                use_ssl=self.conf.EMAIL_USE_SSL,
+                use_tls=self.conf.EMAIL_USE_TLS,
+            )
+
+    def select_queues(self, queues=None):
+        return self.amqp.queues.select_subset(queues)
+
+    def either(self, default_key, *values):
+        """Fallback to the value of a configuration key if none of the
+        `*values` are true."""
+        return first(None, values) or self.conf.get(default_key)
+
+    def bugreport(self):
+        return bugreport(self)
+
+    def _get_backend(self):
+        from celery.backends import get_backend_by_url
+        backend, url = get_backend_by_url(
+            self.backend_cls or self.conf.CELERY_RESULT_BACKEND,
+            self.loader)
+        return backend(app=self, url=url)
+
+    def _get_config(self):
+        self.configured = True
+        s = Settings({}, [self.prepare_config(self.loader.conf),
+                          deepcopy(DEFAULTS)])
+
+        # load lazy config dict initializers.
+        pending = self._pending_defaults
+        while pending:
+            s.add_defaults(pending.popleft()())
+        if self._preconf:
+            for key, value in self._preconf.iteritems():
+                setattr(s, key, value)
+        return s
+
+    def _after_fork(self, obj_):
+        self._maybe_close_pool()
+
+    def _maybe_close_pool(self):
+        if self._pool:
+            self._pool.force_close_all()
+            self._pool = None
+            amqp = self.amqp
+            if amqp._producer_pool:
+                amqp._producer_pool.force_close_all()
+                amqp._producer_pool = None
+
+    def create_task_cls(self):
+        """Creates a base task class using default configuration
+        taken from this app."""
+        return self.subclass_with_self('celery.app.task:Task', name='Task',
+                                       attribute='_app', abstract=True)
+
+    def subclass_with_self(self, Class, name=None, attribute='app',
+                           reverse=None, **kw):
+        """Subclass an app-compatible class by setting its app attribute
+        to be this app instance.
+
+        App-compatible means that the class has a class attribute that
+        provides the default app it should use, e.g.
+        ``class Foo: app = None``.
+
+        :param Class: The app-compatible class to subclass.
+        :keyword name: Custom name for the target class.
+        :keyword attribute: Name of the attribute holding the app,
+                            default is 'app'.
+
+        """
+        Class = symbol_by_name(Class)
+        reverse = reverse if reverse else Class.__name__
+
+        def __reduce__(self):
+            return _unpickle_appattr, (reverse, self.__reduce_args__())
+
+        attrs = dict({attribute: self}, __module__=Class.__module__,
+                     __doc__=Class.__doc__, __reduce__=__reduce__, **kw)
+
+        return type(name or Class.__name__, (Class, ), attrs)
+
+    def _rgetattr(self, path):
+        return reduce(getattr, [self] + path.split('.'))
+
+    def __repr__(self):
+        return '<%s %s:0x%x>' % (self.__class__.__name__,
+                                 self.main or '__main__', id(self), )
+
+    def __reduce__(self):
+        # Reduce only pickles the configuration changes,
+        # so the default configuration doesn't have to be passed
+        # between processes.
+        return (
+            _unpickle_app,
+            (self.__class__, self.Pickler) + self.__reduce_args__(),
+        )
+
+    def __reduce_args__(self):
+        return (self.main, self.conf.changes, self.loader_cls,
+                self.backend_cls, self.amqp_cls, self.events_cls,
+                self.log_cls, self.control_cls, self.accept_magic_kwargs,
+                self._config_source)
+
+    @cached_property
+    def Worker(self):
+        return self.subclass_with_self('celery.apps.worker:Worker')
+
+    @cached_property
+    def WorkController(self, **kwargs):
+        return self.subclass_with_self('celery.worker:WorkController')
+
+    @cached_property
+    def Beat(self, **kwargs):
+        return self.subclass_with_self('celery.apps.beat:Beat')
+
+    @cached_property
+    def TaskSet(self):
+        return self.subclass_with_self('celery.task.sets:TaskSet')
+
+    @cached_property
+    def Task(self):
+        return self.create_task_cls()
+
+    @cached_property
+    def annotations(self):
+        return prepare_annotations(self.conf.CELERY_ANNOTATIONS)
+
+    @cached_property
+    def AsyncResult(self):
+        return self.subclass_with_self('celery.result:AsyncResult')
+
+    @cached_property
+    def GroupResult(self):
+        return self.subclass_with_self('celery.result:GroupResult')
+
+    @cached_property
+    def TaskSetResult(self):  # XXX compat
+        return self.subclass_with_self('celery.result:TaskSetResult')
+
+    @property
+    def pool(self):
+        if self._pool is None:
+            register_after_fork(self, self._after_fork)
+            limit = self.conf.BROKER_POOL_LIMIT
+            self._pool = self.connection().Pool(limit=limit)
+        return self._pool
+
+    @property
+    def current_task(self):
+        return _task_stack.top
+
+    @cached_property
+    def amqp(self):
+        return instantiate(self.amqp_cls, app=self)
+
+    @cached_property
+    def backend(self):
+        return self._get_backend()
+
+    @cached_property
+    def conf(self):
+        return self._get_config()
+
+    @cached_property
+    def control(self):
+        return instantiate(self.control_cls, app=self)
+
+    @cached_property
+    def events(self):
+        return instantiate(self.events_cls, app=self)
+
+    @cached_property
+    def loader(self):
+        return get_loader_cls(self.loader_cls)(app=self)
+
+    @cached_property
+    def log(self):
+        return instantiate(self.log_cls, app=self)
+
+    @cached_property
+    def tasks(self):
+        self.finalize()
+        return self._tasks
+App = Celery  # compat

awx/lib/site-packages/celery/app/builtins.py

@@ -0,0 +1,374 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.builtins
+    ~~~~~~~~~~~~~~~~~~~
+
+    Built-in tasks that are always available in all
+    app instances. E.g. chord, group and xmap.
+
+"""
+from __future__ import absolute_import
+from __future__ import with_statement
+
+from collections import deque
+
+from celery._state import get_current_worker_task
+from celery.utils import uuid
+
+#: global list of functions defining tasks that should be
+#: added to all apps.
+_shared_tasks = []
+
+
+def shared_task(constructor):
+    """Decorator that specifies that the decorated function is a function
+    that generates a built-in task.
+
+    The function will then be called for every new app instance created
+    (lazily, so more exactly when the task registry for that app is needed).
+    """
+    _shared_tasks.append(constructor)
+    return constructor
+
+
+def load_shared_tasks(app):
+    """Loads the built-in tasks for an app instance."""
+    for constructor in _shared_tasks:
+        constructor(app)
+
+
+@shared_task
+def add_backend_cleanup_task(app):
+    """The backend cleanup task can be used to clean up the default result
+    backend.
+
+    This task is also added do the periodic task schedule so that it is
+    run every day at midnight, but :program:`celerybeat` must be running
+    for this to be effective.
+
+    Note that not all backends do anything for this, what needs to be
+    done at cleanup is up to each backend, and some backends
+    may even clean up in realtime so that a periodic cleanup is not necessary.
+
+    """
+
+    @app.task(name='celery.backend_cleanup', _force_evaluate=True)
+    def backend_cleanup():
+        app.backend.cleanup()
+    return backend_cleanup
+
+
+@shared_task
+def add_unlock_chord_task(app):
+    """The unlock chord task is used by result backends that doesn't
+    have native chord support.
+
+    It creates a task chain polling the header for completion.
+
+    """
+    from celery.canvas import subtask
+    from celery.exceptions import ChordError
+    from celery.result import from_serializable
+
+    default_propagate = app.conf.CELERY_CHORD_PROPAGATES
+
+    @app.task(name='celery.chord_unlock', max_retries=None,
+              default_retry_delay=1, ignore_result=True, _force_evaluate=True)
+    def unlock_chord(group_id, callback, interval=None, propagate=None,
+                     max_retries=None, result=None,
+                     Result=app.AsyncResult, GroupResult=app.GroupResult,
+                     from_serializable=from_serializable):
+        # if propagate is disabled exceptions raised by chord tasks
+        # will be sent as part of the result list to the chord callback.
+        # Since 3.1 propagate will be enabled by default, and instead
+        # the chord callback changes state to FAILURE with the
+        # exception set to ChordError.
+        propagate = default_propagate if propagate is None else propagate
+
+        # check if the task group is ready, and if so apply the callback.
+        deps = GroupResult(
+            group_id,
+            [from_serializable(r, app=app) for r in result],
+        )
+        j = deps.join_native if deps.supports_native_join else deps.join
+
+        if deps.ready():
+            callback = subtask(callback)
+            try:
+                ret = j(propagate=propagate)
+            except Exception, exc:
+                try:
+                    culprit = deps._failed_join_report().next()
+                    reason = 'Dependency %s raised %r' % (culprit.id, exc)
+                except StopIteration:
+                    reason = repr(exc)
+                app._tasks[callback.task].backend.fail_from_current_stack(
+                    callback.id, exc=ChordError(reason),
+                )
+            else:
+                try:
+                    callback.delay(ret)
+                except Exception, exc:
+                    app._tasks[callback.task].backend.fail_from_current_stack(
+                        callback.id,
+                        exc=ChordError('Callback error: %r' % (exc, )),
+                    )
+        else:
+            return unlock_chord.retry(countdown=interval,
+                                      max_retries=max_retries)
+    return unlock_chord
+
+
+@shared_task
+def add_map_task(app):
+    from celery.canvas import subtask
+
+    @app.task(name='celery.map', _force_evaluate=True)
+    def xmap(task, it):
+        task = subtask(task).type
+        return [task(value) for value in it]
+    return xmap
+
+
+@shared_task
+def add_starmap_task(app):
+    from celery.canvas import subtask
+
+    @app.task(name='celery.starmap', _force_evaluate=True)
+    def xstarmap(task, it):
+        task = subtask(task).type
+        return [task(*args) for args in it]
+    return xstarmap
+
+
+@shared_task
+def add_chunk_task(app):
+    from celery.canvas import chunks as _chunks
+
+    @app.task(name='celery.chunks', _force_evaluate=True)
+    def chunks(task, it, n):
+        return _chunks.apply_chunks(task, it, n)
+    return chunks
+
+
+@shared_task
+def add_group_task(app):
+    _app = app
+    from celery.canvas import maybe_subtask, subtask
+    from celery.result import from_serializable
+
+    class Group(app.Task):
+        app = _app
+        name = 'celery.group'
+        accept_magic_kwargs = False
+
+        def run(self, tasks, result, group_id, partial_args):
+            app = self.app
+            result = from_serializable(result, app)
+            # any partial args are added to all tasks in the group
+            taskit = (subtask(task).clone(partial_args)
+                      for i, task in enumerate(tasks))
+            if self.request.is_eager or app.conf.CELERY_ALWAYS_EAGER:
+                return app.GroupResult(
+                    result.id,
+                    [stask.apply(group_id=group_id) for stask in taskit],
+                )
+            with app.producer_or_acquire() as pub:
+                [stask.apply_async(group_id=group_id, publisher=pub,
+                                   add_to_parent=False) for stask in taskit]
+            parent = get_current_worker_task()
+            if parent:
+                parent.request.children.append(result)
+            return result
+
+        def prepare(self, options, tasks, args, **kwargs):
+            AsyncResult = self.AsyncResult
+            options['group_id'] = group_id = (
+                options.setdefault('task_id', uuid()))
+
+            def prepare_member(task):
+                task = maybe_subtask(task)
+                opts = task.options
+                opts['group_id'] = group_id
+                try:
+                    tid = opts['task_id']
+                except KeyError:
+                    tid = opts['task_id'] = uuid()
+                return task, AsyncResult(tid)
+
+            try:
+                tasks, results = zip(*[prepare_member(task) for task in tasks])
+            except ValueError:  # tasks empty
+                tasks, results = [], []
+            return (tasks, self.app.GroupResult(group_id, results),
+                    group_id, args)
+
+        def apply_async(self, partial_args=(), kwargs={}, **options):
+            if self.app.conf.CELERY_ALWAYS_EAGER:
+                return self.apply(partial_args, kwargs, **options)
+            tasks, result, gid, args = self.prepare(
+                options, args=partial_args, **kwargs
+            )
+            super(Group, self).apply_async((
+                list(tasks), result.serializable(), gid, args), **options
+            )
+            return result
+
+        def apply(self, args=(), kwargs={}, **options):
+            return super(Group, self).apply(
+                self.prepare(options, args=args, **kwargs),
+                **options).get()
+    return Group
+
+
+@shared_task
+def add_chain_task(app):
+    from celery.canvas import Signature, chord, group, maybe_subtask
+    _app = app
+
+    class Chain(app.Task):
+        app = _app
+        name = 'celery.chain'
+        accept_magic_kwargs = False
+
+        def prepare_steps(self, args, tasks):
+            steps = deque(tasks)
+            next_step = prev_task = prev_res = None
+            tasks, results = [], []
+            i = 0
+            while steps:
+                # First task get partial args from chain.
+                task = maybe_subtask(steps.popleft())
+                task = task.clone() if i else task.clone(args)
+                res = task._freeze()
+                i += 1
+
+                if isinstance(task, group):
+                    # automatically upgrade group(..) | s to chord(group, s)
+                    try:
+                        next_step = steps.popleft()
+                        # for chords we freeze by pretending it's a normal
+                        # task instead of a group.
+                        res = Signature._freeze(task)
+                        task = chord(task, body=next_step, task_id=res.task_id)
+                    except IndexError:
+                        pass
+                if prev_task:
+                    # link previous task to this task.
+                    prev_task.link(task)
+                    # set the results parent attribute.
+                    res.parent = prev_res
+
+                results.append(res)
+                tasks.append(task)
+                prev_task, prev_res = task, res
+
+            return tasks, results
+
+        def apply_async(self, args=(), kwargs={}, group_id=None, chord=None,
+                        task_id=None, **options):
+            if self.app.conf.CELERY_ALWAYS_EAGER:
+                return self.apply(args, kwargs, **options)
+            options.pop('publisher', None)
+            tasks, results = self.prepare_steps(args, kwargs['tasks'])
+            result = results[-1]
+            if group_id:
+                tasks[-1].set(group_id=group_id)
+            if chord:
+                tasks[-1].set(chord=chord)
+            if task_id:
+                tasks[-1].set(task_id=task_id)
+                result = tasks[-1].type.AsyncResult(task_id)
+            tasks[0].apply_async()
+            return result
+
+        def apply(self, args=(), kwargs={}, subtask=maybe_subtask, **options):
+            last, fargs = None, args  # fargs passed to first task only
+            for task in kwargs['tasks']:
+                res = subtask(task).clone(fargs).apply(last and (last.get(), ))
+                res.parent, last, fargs = last, res, None
+            return last
+    return Chain
+
+
+@shared_task
+def add_chord_task(app):
+    """Every chord is executed in a dedicated task, so that the chord
+    can be used as a subtask, and this generates the task
+    responsible for that."""
+    from celery import group
+    from celery.canvas import maybe_subtask
+    _app = app
+    default_propagate = app.conf.CELERY_CHORD_PROPAGATES
+
+    class Chord(app.Task):
+        app = _app
+        name = 'celery.chord'
+        accept_magic_kwargs = False
+        ignore_result = False
+
+        def run(self, header, body, partial_args=(), interval=1, countdown=1,
+                max_retries=None, propagate=None, eager=False, **kwargs):
+            propagate = default_propagate if propagate is None else propagate
+            group_id = uuid()
+            AsyncResult = self.app.AsyncResult
+            prepare_member = self._prepare_member
+
+            # - convert back to group if serialized
+            tasks = header.tasks if isinstance(header, group) else header
+            header = group([maybe_subtask(s).clone() for s in tasks])
+            # - eager applies the group inline
+            if eager:
+                return header.apply(args=partial_args, task_id=group_id)
+
+            results = [AsyncResult(prepare_member(task, body, group_id))
+                       for task in header.tasks]
+
+            # - fallback implementations schedules the chord_unlock task here
+            app.backend.on_chord_apply(group_id, body,
+                                       interval=interval,
+                                       countdown=countdown,
+                                       max_retries=max_retries,
+                                       propagate=propagate,
+                                       result=results)
+            # - call the header group, returning the GroupResult.
+            # XXX Python 2.5 doesn't allow kwargs after star-args.
+            return header(*partial_args, **{'task_id': group_id})
+
+        def _prepare_member(self, task, body, group_id):
+            opts = task.options
+            # d.setdefault would work but generating uuid's are expensive
+            try:
+                task_id = opts['task_id']
+            except KeyError:
+                task_id = opts['task_id'] = uuid()
+            opts.update(chord=body, group_id=group_id)
+            return task_id
+
+        def apply_async(self, args=(), kwargs={}, task_id=None, **options):
+            if self.app.conf.CELERY_ALWAYS_EAGER:
+                return self.apply(args, kwargs, **options)
+            group_id = options.pop('group_id', None)
+            chord = options.pop('chord', None)
+            header = kwargs.pop('header')
+            body = kwargs.pop('body')
+            header, body = (list(maybe_subtask(header)),
+                            maybe_subtask(body))
+            if group_id:
+                body.set(group_id=group_id)
+            if chord:
+                body.set(chord=chord)
+            callback_id = body.options.setdefault('task_id', task_id or uuid())
+            parent = super(Chord, self).apply_async((header, body, args),
+                                                    kwargs, **options)
+            body_result = self.AsyncResult(callback_id)
+            body_result.parent = parent
+            return body_result
+
+        def apply(self, args=(), kwargs={}, propagate=True, **options):
+            body = kwargs['body']
+            res = super(Chord, self).apply(args, dict(kwargs, eager=True),
+                                           **options)
+            return maybe_subtask(body).apply(
+                args=(res.get(propagate=propagate).get(), ))
+    return Chord

awx/lib/site-packages/celery/app/control.py

@@ -0,0 +1,270 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.control
+    ~~~~~~~~~~~~~~~~~~~
+
+    Client for worker remote control commands.
+    Server implementation is in :mod:`celery.worker.control`.
+
+"""
+from __future__ import absolute_import
+from __future__ import with_statement
+
+from kombu.pidbox import Mailbox
+from kombu.utils import cached_property
+
+from . import app_or_default
+
+
+def flatten_reply(reply):
+    nodes = {}
+    for item in reply:
+        nodes.update(item)
+    return nodes
+
+
+class Inspect(object):
+    app = None
+
+    def __init__(self, destination=None, timeout=1, callback=None,
+                 connection=None, app=None, limit=None):
+        self.app = app or self.app
+        self.destination = destination
+        self.timeout = timeout
+        self.callback = callback
+        self.connection = connection
+        self.limit = limit
+
+    def _prepare(self, reply):
+        if not reply:
+            return
+        by_node = flatten_reply(reply)
+        if self.destination and \
+                not isinstance(self.destination, (list, tuple)):
+            return by_node.get(self.destination)
+        return by_node
+
+    def _request(self, command, **kwargs):
+        return self._prepare(self.app.control.broadcast(
+            command,
+            arguments=kwargs,
+            destination=self.destination,
+            callback=self.callback,
+            connection=self.connection,
+            limit=self.limit,
+            timeout=self.timeout, reply=True,
+        ))
+
+    def report(self):
+        return self._request('report')
+
+    def active(self, safe=False):
+        return self._request('dump_active', safe=safe)
+
+    def scheduled(self, safe=False):
+        return self._request('dump_schedule', safe=safe)
+
+    def reserved(self, safe=False):
+        return self._request('dump_reserved', safe=safe)
+
+    def stats(self):
+        return self._request('stats')
+
+    def revoked(self):
+        return self._request('dump_revoked')
+
+    def registered(self, *taskinfoitems):
+        return self._request('dump_tasks', taskinfoitems=taskinfoitems)
+    registered_tasks = registered
+
+    def ping(self):
+        return self._request('ping')
+
+    def active_queues(self):
+        return self._request('active_queues')
+
+    def conf(self):
+        return self._request('dump_conf')
+
+
+class Control(object):
+    Mailbox = Mailbox
+
+    def __init__(self, app=None):
+        self.app = app_or_default(app)
+        self.mailbox = self.Mailbox('celery', type='fanout',
+                                    accept=self.app.conf.CELERY_ACCEPT_CONTENT)
+
+    @cached_property
+    def inspect(self):
+        return self.app.subclass_with_self(Inspect, reverse='control.inspect')
+
+    def purge(self, connection=None):
+        """Discard all waiting tasks.
+
+        This will ignore all tasks waiting for execution, and they will
+        be deleted from the messaging server.
+
+        :returns: the number of tasks discarded.
+
+        """
+        with self.app.connection_or_acquire(connection) as conn:
+            return self.app.amqp.TaskConsumer(conn).purge()
+    discard_all = purge
+
+    def revoke(self, task_id, destination=None, terminate=False,
+               signal='SIGTERM', **kwargs):
+        """Tell all (or specific) workers to revoke a task by id.
+
+        If a task is revoked, the workers will ignore the task and
+        not execute it after all.
+
+        :param task_id: Id of the task to revoke.
+        :keyword terminate: Also terminate the process currently working
+            on the task (if any).
+        :keyword signal: Name of signal to send to process if terminate.
+            Default is TERM.
+
+        See :meth:`broadcast` for supported keyword arguments.
+
+        """
+        return self.broadcast('revoke', destination=destination,
+                              arguments={'task_id': task_id,
+                                         'terminate': terminate,
+                                         'signal': signal}, **kwargs)
+
+    def ping(self, destination=None, timeout=1, **kwargs):
+        """Ping all (or specific) workers.
+
+        Returns answer from alive workers.
+
+        See :meth:`broadcast` for supported keyword arguments.
+
+        """
+        return self.broadcast('ping', reply=True, destination=destination,
+                              timeout=timeout, **kwargs)
+
+    def rate_limit(self, task_name, rate_limit, destination=None, **kwargs):
+        """Tell all (or specific) workers to set a new rate limit
+        for task by type.
+
+        :param task_name: Name of task to change rate limit for.
+        :param rate_limit: The rate limit as tasks per second, or a rate limit
+            string (`'100/m'`, etc.
+            see :attr:`celery.task.base.Task.rate_limit` for
+            more information).
+
+        See :meth:`broadcast` for supported keyword arguments.
+
+        """
+        return self.broadcast('rate_limit', destination=destination,
+                              arguments={'task_name': task_name,
+                                         'rate_limit': rate_limit},
+                              **kwargs)
+
+    def add_consumer(self, queue, exchange=None, exchange_type='direct',
+                     routing_key=None, options=None, **kwargs):
+        """Tell all (or specific) workers to start consuming from a new queue.
+
+        Only the queue name is required as if only the queue is specified
+        then the exchange/routing key will be set to the same name (
+        like automatic queues do).
+
+        .. note::
+
+            This command does not respect the default queue/exchange
+            options in the configuration.
+
+        :param queue: Name of queue to start consuming from.
+        :keyword exchange: Optional name of exchange.
+        :keyword exchange_type: Type of exchange (defaults to 'direct')
+            command to, when empty broadcast to all workers.
+        :keyword routing_key: Optional routing key.
+        :keyword options: Additional options as supported
+            by :meth:`kombu.entitiy.Queue.from_dict`.
+
+        See :meth:`broadcast` for supported keyword arguments.
+
+        """
+        return self.broadcast(
+            'add_consumer',
+            arguments=dict({'queue': queue, 'exchange': exchange,
+                            'exchange_type': exchange_type,
+                            'routing_key': routing_key}, **options or {}),
+            **kwargs
+        )
+
+    def cancel_consumer(self, queue, **kwargs):
+        """Tell all (or specific) workers to stop consuming from ``queue``.
+
+        Supports the same keyword arguments as :meth:`broadcast`.
+
+        """
+        return self.broadcast(
+            'cancel_consumer', arguments={'queue': queue}, **kwargs
+        )
+
+    def time_limit(self, task_name, soft=None, hard=None, **kwargs):
+        """Tell all (or specific) workers to set time limits for
+        a task by type.
+
+        :param task_name: Name of task to change time limits for.
+        :keyword soft: New soft time limit (in seconds).
+        :keyword hard: New hard time limit (in seconds).
+
+        Any additional keyword arguments are passed on to :meth:`broadcast`.
+
+        """
+        return self.broadcast(
+            'time_limit',
+            arguments={'task_name': task_name,
+                       'hard': hard, 'soft': soft}, **kwargs)
+
+    def enable_events(self, destination=None, **kwargs):
+        """Tell all (or specific) workers to enable events."""
+        return self.broadcast('enable_events', {}, destination, **kwargs)
+
+    def disable_events(self, destination=None, **kwargs):
+        """Tell all (or specific) workers to enable events."""
+        return self.broadcast('disable_events', {}, destination, **kwargs)
+
+    def pool_grow(self, n=1, destination=None, **kwargs):
+        """Tell all (or specific) workers to grow the pool by ``n``.
+
+        Supports the same arguments as :meth:`broadcast`.
+
+        """
+        return self.broadcast('pool_grow', {}, destination, **kwargs)
+
+    def pool_shrink(self, n=1, destination=None, **kwargs):
+        """Tell all (or specific) workers to shrink the pool by ``n``.
+
+        Supports the same arguments as :meth:`broadcast`.
+
+        """
+        return self.broadcast('pool_shrink', {}, destination, **kwargs)
+
+    def broadcast(self, command, arguments=None, destination=None,
+                  connection=None, reply=False, timeout=1, limit=None,
+                  callback=None, channel=None, **extra_kwargs):
+        """Broadcast a control command to the celery workers.
+
+        :param command: Name of command to send.
+        :param arguments: Keyword arguments for the command.
+        :keyword destination: If set, a list of the hosts to send the
+            command to, when empty broadcast to all workers.
+        :keyword connection: Custom broker connection to use, if not set,
+            a connection will be established automatically.
+        :keyword reply: Wait for and return the reply.
+        :keyword timeout: Timeout in seconds to wait for the reply.
+        :keyword limit: Limit number of replies.
+        :keyword callback: Callback called immediately for each reply
+            received.
+
+        """
+        with self.app.connection_or_acquire(connection) as conn:
+            arguments = dict(arguments or {}, **extra_kwargs)
+            return self.mailbox(conn)._broadcast(
+                command, arguments, destination, reply, timeout,
+                limit, callback, channel=channel,
+            )

awx/lib/site-packages/celery/app/defaults.py

@@ -0,0 +1,267 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.defaults
+    ~~~~~~~~~~~~~~~~~~~
+
+    Configuration introspection and defaults.
+
+"""
+from __future__ import absolute_import
+
+import sys
+
+from collections import deque
+from datetime import timedelta
+
+from celery.utils import strtobool
+from celery.utils.functional import memoize
+
+is_jython = sys.platform.startswith('java')
+is_pypy = hasattr(sys, 'pypy_version_info')
+
+DEFAULT_POOL = 'processes'
+if is_jython:
+    DEFAULT_POOL = 'threads'
+elif is_pypy:
+    if sys.pypy_version_info[0:3] < (1, 5, 0):
+        DEFAULT_POOL = 'solo'
+    else:
+        DEFAULT_POOL = 'processes'
+
+
+DEFAULT_PROCESS_LOG_FMT = """
+    [%(asctime)s: %(levelname)s/%(processName)s] %(message)s
+""".strip()
+DEFAULT_LOG_FMT = '[%(asctime)s: %(levelname)s] %(message)s'
+DEFAULT_TASK_LOG_FMT = """[%(asctime)s: %(levelname)s/%(processName)s] \
+%(task_name)s[%(task_id)s]: %(message)s"""
+
+_BROKER_OLD = {'deprecate_by': '2.5', 'remove_by': '4.0', 'alt': 'BROKER_URL'}
+_REDIS_OLD = {'deprecate_by': '2.5', 'remove_by': '4.0',
+              'alt': 'URL form of CELERY_RESULT_BACKEND'}
+
+
+class Option(object):
+    alt = None
+    deprecate_by = None
+    remove_by = None
+    typemap = dict(string=str, int=int, float=float, any=lambda v: v,
+                   bool=strtobool, dict=dict, tuple=tuple)
+
+    def __init__(self, default=None, *args, **kwargs):
+        self.default = default
+        self.type = kwargs.get('type') or 'string'
+        for attr, value in kwargs.iteritems():
+            setattr(self, attr, value)
+
+    def to_python(self, value):
+        return self.typemap[self.type](value)
+
+    def __repr__(self):
+        return '<Option: type->%s default->%r>' % (self.type, self.default)
+
+
+NAMESPACES = {
+    'BROKER': {
+        'URL': Option(None, type='string'),
+        'CONNECTION_TIMEOUT': Option(4, type='float'),
+        'CONNECTION_RETRY': Option(True, type='bool'),
+        'CONNECTION_MAX_RETRIES': Option(100, type='int'),
+        'HEARTBEAT': Option(None, type='int'),
+        'HEARTBEAT_CHECKRATE': Option(3.0, type='int'),
+        'POOL_LIMIT': Option(10, type='int'),
+        'INSIST': Option(False, type='bool',
+                         deprecate_by='2.4', remove_by='4.0'),
+        'USE_SSL': Option(False, type='bool'),
+        'TRANSPORT': Option(type='string'),
+        'TRANSPORT_OPTIONS': Option({}, type='dict'),
+        'HOST': Option(type='string', **_BROKER_OLD),
+        'PORT': Option(type='int', **_BROKER_OLD),
+        'USER': Option(type='string', **_BROKER_OLD),
+        'PASSWORD': Option(type='string', **_BROKER_OLD),
+        'VHOST': Option(type='string', **_BROKER_OLD),
+    },
+    'CASSANDRA': {
+        'COLUMN_FAMILY': Option(type='string'),
+        'DETAILED_MODE': Option(False, type='bool'),
+        'KEYSPACE': Option(type='string'),
+        'READ_CONSISTENCY': Option(type='string'),
+        'SERVERS': Option(type='list'),
+        'WRITE_CONSISTENCY': Option(type='string'),
+    },
+    'CELERY': {
+        'ACCEPT_CONTENT': Option(None, type='any'),
+        'ACKS_LATE': Option(False, type='bool'),
+        'ALWAYS_EAGER': Option(False, type='bool'),
+        'AMQP_TASK_RESULT_EXPIRES': Option(
+            type='float', deprecate_by='2.5', remove_by='4.0',
+            alt='CELERY_TASK_RESULT_EXPIRES'
+        ),
+        'AMQP_TASK_RESULT_CONNECTION_MAX': Option(
+            1, type='int', remove_by='2.5', alt='BROKER_POOL_LIMIT',
+        ),
+        'ANNOTATIONS': Option(type='any'),
+        'BROADCAST_QUEUE': Option('celeryctl'),
+        'BROADCAST_EXCHANGE': Option('celeryctl'),
+        'BROADCAST_EXCHANGE_TYPE': Option('fanout'),
+        'CACHE_BACKEND': Option(),
+        'CACHE_BACKEND_OPTIONS': Option({}, type='dict'),
+        # chord propagate will be True from v3.1
+        'CHORD_PROPAGATES': Option(False, type='bool'),
+        'CREATE_MISSING_QUEUES': Option(True, type='bool'),
+        'DEFAULT_RATE_LIMIT': Option(type='string'),
+        'DISABLE_RATE_LIMITS': Option(False, type='bool'),
+        'DEFAULT_ROUTING_KEY': Option('celery'),
+        'DEFAULT_QUEUE': Option('celery'),
+        'DEFAULT_EXCHANGE': Option('celery'),
+        'DEFAULT_EXCHANGE_TYPE': Option('direct'),
+        'DEFAULT_DELIVERY_MODE': Option(2, type='string'),
+        'EAGER_PROPAGATES_EXCEPTIONS': Option(False, type='bool'),
+        'ENABLE_UTC': Option(True, type='bool'),
+        'EVENT_SERIALIZER': Option('json'),
+        'IMPORTS': Option((), type='tuple'),
+        'INCLUDE': Option((), type='tuple'),
+        'IGNORE_RESULT': Option(False, type='bool'),
+        'MAX_CACHED_RESULTS': Option(5000, type='int'),
+        'MESSAGE_COMPRESSION': Option(type='string'),
+        'MONGODB_BACKEND_SETTINGS': Option(type='dict'),
+        'REDIS_HOST': Option(type='string', **_REDIS_OLD),
+        'REDIS_PORT': Option(type='int', **_REDIS_OLD),
+        'REDIS_DB': Option(type='int', **_REDIS_OLD),
+        'REDIS_PASSWORD': Option(type='string', **_REDIS_OLD),
+        'REDIS_MAX_CONNECTIONS': Option(type='int'),
+        'RESULT_BACKEND': Option(type='string'),
+        'RESULT_DB_SHORT_LIVED_SESSIONS': Option(False, type='bool'),
+        'RESULT_DBURI': Option(),
+        'RESULT_ENGINE_OPTIONS': Option(type='dict'),
+        'RESULT_EXCHANGE': Option('celeryresults'),
+        'RESULT_EXCHANGE_TYPE': Option('direct'),
+        'RESULT_SERIALIZER': Option('pickle'),
+        'RESULT_PERSISTENT': Option(False, type='bool'),
+        'ROUTES': Option(type='any'),
+        'SEND_EVENTS': Option(False, type='bool'),
+        'SEND_TASK_ERROR_EMAILS': Option(False, type='bool'),
+        'SEND_TASK_SENT_EVENT': Option(False, type='bool'),
+        'STORE_ERRORS_EVEN_IF_IGNORED': Option(False, type='bool'),
+        'TASK_ERROR_WHITELIST': Option(
+            (), type='tuple', deprecate_by='2.5', remove_by='4.0',
+        ),
+        'TASK_PUBLISH_RETRY': Option(True, type='bool'),
+        'TASK_PUBLISH_RETRY_POLICY': Option({
+            'max_retries': 3,
+            'interval_start': 0,
+            'interval_max': 1,
+            'interval_step': 0.2}, type='dict'),
+        'TASK_RESULT_EXPIRES': Option(timedelta(days=1), type='float'),
+        'TASK_SERIALIZER': Option('pickle'),
+        'TIMEZONE': Option(type='string'),
+        'TRACK_STARTED': Option(False, type='bool'),
+        'REDIRECT_STDOUTS': Option(True, type='bool'),
+        'REDIRECT_STDOUTS_LEVEL': Option('WARNING'),
+        'QUEUES': Option(type='dict'),
+        'QUEUE_HA_POLICY': Option(None, type='string'),
+        'SECURITY_KEY': Option(type='string'),
+        'SECURITY_CERTIFICATE': Option(type='string'),
+        'SECURITY_CERT_STORE': Option(type='string'),
+        'WORKER_DIRECT': Option(False, type='bool'),
+    },
+    'CELERYD': {
+        'AUTOSCALER': Option('celery.worker.autoscale.Autoscaler'),
+        'AUTORELOADER': Option('celery.worker.autoreload.Autoreloader'),
+        'BOOT_STEPS': Option((), type='tuple'),
+        'CONCURRENCY': Option(0, type='int'),
+        'TIMER': Option(type='string'),
+        'TIMER_PRECISION': Option(1.0, type='float'),
+        'FORCE_EXECV': Option(False, type='bool'),
+        'HIJACK_ROOT_LOGGER': Option(True, type='bool'),
+        'CONSUMER': Option(type='string'),
+        'LOG_FORMAT': Option(DEFAULT_PROCESS_LOG_FMT),
+        'LOG_COLOR': Option(type='bool'),
+        'LOG_LEVEL': Option('WARN', deprecate_by='2.4', remove_by='4.0',
+                            alt='--loglevel argument'),
+        'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0',
+                           alt='--logfile argument'),
+        'MEDIATOR': Option('celery.worker.mediator.Mediator'),
+        'MAX_TASKS_PER_CHILD': Option(type='int'),
+        'POOL': Option(DEFAULT_POOL),
+        'POOL_PUTLOCKS': Option(True, type='bool'),
+        'POOL_RESTARTS': Option(False, type='bool'),
+        'PREFETCH_MULTIPLIER': Option(4, type='int'),
+        'STATE_DB': Option(),
+        'TASK_LOG_FORMAT': Option(DEFAULT_TASK_LOG_FMT),
+        'TASK_SOFT_TIME_LIMIT': Option(type='float'),
+        'TASK_TIME_LIMIT': Option(type='float'),
+        'WORKER_LOST_WAIT': Option(10.0, type='float')
+    },
+    'CELERYBEAT': {
+        'SCHEDULE': Option({}, type='dict'),
+        'SCHEDULER': Option('celery.beat.PersistentScheduler'),
+        'SCHEDULE_FILENAME': Option('celerybeat-schedule'),
+        'MAX_LOOP_INTERVAL': Option(0, type='float'),
+        'LOG_LEVEL': Option('INFO', deprecate_by='2.4', remove_by='4.0',
+                            alt='--loglevel argument'),
+        'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0',
+                           alt='--logfile argument'),
+    },
+    'CELERYMON': {
+        'LOG_LEVEL': Option('INFO', deprecate_by='2.4', remove_by='4.0',
+                            alt='--loglevel argument'),
+        'LOG_FILE': Option(deprecate_by='2.4', remove_by='4.0',
+                           alt='--logfile argument'),
+        'LOG_FORMAT': Option(DEFAULT_LOG_FMT),
+    },
+    'EMAIL': {
+        'HOST': Option('localhost'),
+        'PORT': Option(25, type='int'),
+        'HOST_USER': Option(),
+        'HOST_PASSWORD': Option(),
+        'TIMEOUT': Option(2, type='float'),
+        'USE_SSL': Option(False, type='bool'),
+        'USE_TLS': Option(False, type='bool'),
+    },
+    'SERVER_EMAIL': Option('celery@localhost'),
+    'ADMINS': Option((), type='tuple'),
+}
+
+
+def flatten(d, ns=''):
+    stack = deque([(ns, d)])
+    while stack:
+        name, space = stack.popleft()
+        for key, value in space.iteritems():
+            if isinstance(value, dict):
+                stack.append((name + key + '_', value))
+            else:
+                yield name + key, value
+DEFAULTS = dict((key, value.default) for key, value in flatten(NAMESPACES))
+
+
+def find_deprecated_settings(source):
+    from celery.utils import warn_deprecated
+    for name, opt in flatten(NAMESPACES):
+        if (opt.deprecate_by or opt.remove_by) and getattr(source, name, None):
+            warn_deprecated(description='The %r setting' % (name, ),
+                            deprecation=opt.deprecate_by,
+                            removal=opt.remove_by,
+                            alternative='Use %s instead' % (opt.alt, ))
+    return source
+
+
+@memoize(maxsize=None)
+def find(name, namespace='celery'):
+    # - Try specified namespace first.
+    namespace = namespace.upper()
+    try:
+        return namespace, name.upper(), NAMESPACES[namespace][name.upper()]
+    except KeyError:
+        # - Try all the other namespaces.
+        for ns, keys in NAMESPACES.iteritems():
+            if ns.upper() == name.upper():
+                return None, ns, keys
+            elif isinstance(keys, dict):
+                try:
+                    return ns, name.upper(), keys[name.upper()]
+                except KeyError:
+                    pass
+    # - See if name is a qualname last.
+    return None, name.upper(), DEFAULTS[name.upper()]

awx/lib/site-packages/celery/app/log.py

@@ -0,0 +1,231 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.log
+    ~~~~~~~~~~~~~~
+
+    The Celery instances logging section: ``Celery.log``.
+
+    Sets up logging for the worker and other programs,
+    redirects stdouts, colors log output, patches logging
+    related compatibility fixes, and so on.
+
+"""
+from __future__ import absolute_import
+
+import logging
+import os
+import sys
+
+from kombu.log import NullHandler
+
+from celery import signals
+from celery._state import get_current_task
+from celery.utils import isatty
+from celery.utils.compat import WatchedFileHandler
+from celery.utils.log import (
+    get_logger, mlevel,
+    ColorFormatter, ensure_process_aware_logger,
+    LoggingProxy, get_multiprocessing_logger,
+    reset_multiprocessing_logger,
+)
+from celery.utils.term import colored
+
+is_py3k = sys.version_info[0] == 3
+
+MP_LOG = os.environ.get('MP_LOG', False)
+
+
+class TaskFormatter(ColorFormatter):
+
+    def format(self, record):
+        task = get_current_task()
+        if task and task.request:
+            record.__dict__.update(task_id=task.request.id,
+                                   task_name=task.name)
+        else:
+            record.__dict__.setdefault('task_name', '???')
+            record.__dict__.setdefault('task_id', '???')
+        return ColorFormatter.format(self, record)
+
+
+class Logging(object):
+    #: The logging subsystem is only configured once per process.
+    #: setup_logging_subsystem sets this flag, and subsequent calls
+    #: will do nothing.
+    _setup = False
+
+    def __init__(self, app):
+        self.app = app
+        self.loglevel = mlevel(self.app.conf.CELERYD_LOG_LEVEL)
+        self.format = self.app.conf.CELERYD_LOG_FORMAT
+        self.task_format = self.app.conf.CELERYD_TASK_LOG_FORMAT
+        self.colorize = self.app.conf.CELERYD_LOG_COLOR
+
+    def setup(self, loglevel=None, logfile=None, redirect_stdouts=False,
+              redirect_level='WARNING', colorize=None):
+        handled = self.setup_logging_subsystem(
+            loglevel, logfile, colorize=colorize,
+        )
+        if not handled:
+            logger = get_logger('celery.redirected')
+            if redirect_stdouts:
+                self.redirect_stdouts_to_logger(logger,
+                                                loglevel=redirect_level)
+        os.environ.update(
+            CELERY_LOG_LEVEL=str(loglevel) if loglevel else '',
+            CELERY_LOG_FILE=str(logfile) if logfile else '',
+            CELERY_LOG_REDIRECT='1' if redirect_stdouts else '',
+            CELERY_LOG_REDIRECT_LEVEL=str(redirect_level),
+        )
+
+    def setup_logging_subsystem(self, loglevel=None, logfile=None,
+                                format=None, colorize=None, **kwargs):
+        if Logging._setup:
+            return
+        Logging._setup = True
+        loglevel = mlevel(loglevel or self.loglevel)
+        format = format or self.format
+        colorize = self.supports_color(colorize, logfile)
+        reset_multiprocessing_logger()
+        if not is_py3k:
+            ensure_process_aware_logger()
+        receivers = signals.setup_logging.send(
+            sender=None, loglevel=loglevel, logfile=logfile,
+            format=format, colorize=colorize,
+        )
+
+        if not receivers:
+            root = logging.getLogger()
+
+            if self.app.conf.CELERYD_HIJACK_ROOT_LOGGER:
+                root.handlers = []
+
+            # Configure root logger
+            self._configure_logger(
+                root, logfile, loglevel, format, colorize, **kwargs
+            )
+
+            # Configure the multiprocessing logger
+            self._configure_logger(
+                get_multiprocessing_logger(),
+                logfile, loglevel if MP_LOG else logging.ERROR,
+                format, colorize, **kwargs
+            )
+
+            signals.after_setup_logger.send(
+                sender=None, logger=root,
+                loglevel=loglevel, logfile=logfile,
+                format=format, colorize=colorize,
+            )
+
+            # then setup the root task logger.
+            self.setup_task_loggers(loglevel, logfile, colorize=colorize)
+
+        # This is a hack for multiprocessing's fork+exec, so that
+        # logging before Process.run works.
+        logfile_name = logfile if isinstance(logfile, basestring) else ''
+        os.environ.update(
+            _MP_FORK_LOGLEVEL_=str(loglevel),
+            _MP_FORK_LOGFILE_=logfile_name,
+            _MP_FORK_LOGFORMAT_=format,
+        )
+        return receivers
+
+    def _configure_logger(self, logger, logfile, loglevel,
+                          format, colorize, **kwargs):
+        if logger is not None:
+            self.setup_handlers(logger, logfile, format,
+                                colorize, **kwargs)
+            if loglevel:
+                logger.setLevel(loglevel)
+
+    def setup_task_loggers(self, loglevel=None, logfile=None, format=None,
+                           colorize=None, propagate=False, **kwargs):
+        """Setup the task logger.
+
+        If `logfile` is not specified, then `sys.stderr` is used.
+
+        Returns logger object.
+
+        """
+        loglevel = mlevel(loglevel or self.loglevel)
+        format = format or self.task_format
+        colorize = self.supports_color(colorize, logfile)
+
+        logger = self.setup_handlers(
+            get_logger('celery.task'),
+            logfile, format, colorize,
+            formatter=TaskFormatter, **kwargs
+        )
+        logger.setLevel(loglevel)
+        logger.propagate = int(propagate)    # this is an int for some reason.
+                                             # better to not question why.
+        signals.after_setup_task_logger.send(
+            sender=None, logger=logger,
+            loglevel=loglevel, logfile=logfile,
+            format=format, colorize=colorize,
+        )
+        return logger
+
+    def redirect_stdouts_to_logger(self, logger, loglevel=None,
+                                   stdout=True, stderr=True):
+        """Redirect :class:`sys.stdout` and :class:`sys.stderr` to a
+        logging instance.
+
+        :param logger: The :class:`logging.Logger` instance to redirect to.
+        :param loglevel: The loglevel redirected messages will be logged as.
+
+        """
+        proxy = LoggingProxy(logger, loglevel)
+        if stdout:
+            sys.stdout = proxy
+        if stderr:
+            sys.stderr = proxy
+        return proxy
+
+    def supports_color(self, colorize=None, logfile=None):
+        colorize = self.colorize if colorize is None else colorize
+        if self.app.IS_WINDOWS:
+            # Windows does not support ANSI color codes.
+            return False
+        if colorize or colorize is None:
+            # Only use color if there is no active log file
+            # and stderr is an actual terminal.
+            return logfile is None and isatty(sys.stderr)
+        return colorize
+
+    def colored(self, logfile=None, enabled=None):
+        return colored(enabled=self.supports_color(enabled, logfile))
+
+    def setup_handlers(self, logger, logfile, format, colorize,
+                       formatter=ColorFormatter, **kwargs):
+        if self._is_configured(logger):
+            return logger
+        handler = self._detect_handler(logfile)
+        handler.setFormatter(formatter(format, use_color=colorize))
+        logger.addHandler(handler)
+        return logger
+
+    def _detect_handler(self, logfile=None):
+        """Create log handler with either a filename, an open stream
+        or :const:`None` (stderr)."""
+        logfile = sys.__stderr__ if logfile is None else logfile
+        if hasattr(logfile, 'write'):
+            return logging.StreamHandler(logfile)
+        return WatchedFileHandler(logfile)
+
+    def _has_handler(self, logger):
+        return (logger.handlers and
+                not isinstance(logger.handlers[0], NullHandler))
+
+    def _is_configured(self, logger):
+        return self._has_handler(logger) and not getattr(
+            logger, '_rudimentary_setup', False)
+
+    def setup_logger(self, name='celery', *args, **kwargs):
+        """Deprecated: No longer used."""
+        self.setup_logging_subsystem(*args, **kwargs)
+        return logging.root
+
+    def get_default_logger(self, name='celery', **kwargs):
+        return get_logger(name)

awx/lib/site-packages/celery/app/registry.py

@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.registry
+    ~~~~~~~~~~~~~~~~~~~
+
+    Registry of available tasks.
+
+"""
+from __future__ import absolute_import
+
+import inspect
+
+from celery.exceptions import NotRegistered
+
+
+class TaskRegistry(dict):
+    NotRegistered = NotRegistered
+
+    def __missing__(self, key):
+        raise self.NotRegistered(key)
+
+    def register(self, task):
+        """Register a task in the task registry.
+
+        The task will be automatically instantiated if not already an
+        instance.
+
+        """
+        self[task.name] = inspect.isclass(task) and task() or task
+
+    def unregister(self, name):
+        """Unregister task by name.
+
+        :param name: name of the task to unregister, or a
+            :class:`celery.task.base.Task` with a valid `name` attribute.
+
+        :raises celery.exceptions.NotRegistered: if the task has not
+            been registered.
+
+        """
+        try:
+            self.pop(getattr(name, 'name', name))
+        except KeyError:
+            raise self.NotRegistered(name)
+
+    # -- these methods are irrelevant now and will be removed in 4.0
+    def regular(self):
+        return self.filter_types('regular')
+
+    def periodic(self):
+        return self.filter_types('periodic')
+
+    def filter_types(self, type):
+        return dict((name, task) for name, task in self.iteritems()
+                    if getattr(task, 'type', 'regular') == type)
+
+
+def _unpickle_task(name):
+    from celery import current_app
+    return current_app.tasks[name]

awx/lib/site-packages/celery/app/routes.py

@@ -0,0 +1,94 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.routes
+    ~~~~~~~~~~~~~
+
+    Contains utilities for working with task routers,
+    (:setting:`CELERY_ROUTES`).
+
+"""
+from __future__ import absolute_import
+
+from celery.exceptions import QueueNotFound
+from celery.utils import lpmerge
+from celery.utils.functional import firstmethod, mpromise
+from celery.utils.imports import instantiate
+
+_first_route = firstmethod('route_for_task')
+
+
+class MapRoute(object):
+    """Creates a router out of a :class:`dict`."""
+
+    def __init__(self, map):
+        self.map = map
+
+    def route_for_task(self, task, *args, **kwargs):
+        route = self.map.get(task)
+        if route:
+            return dict(route)
+
+
+class Router(object):
+
+    def __init__(self, routes=None, queues=None,
+                 create_missing=False, app=None):
+        self.app = app
+        self.queues = {} if queues is None else queues
+        self.routes = [] if routes is None else routes
+        self.create_missing = create_missing
+
+    def route(self, options, task, args=(), kwargs={}):
+        options = self.expand_destination(options)  # expands 'queue'
+        if self.routes:
+            route = self.lookup_route(task, args, kwargs)
+            if route:  # expands 'queue' in route.
+                return lpmerge(self.expand_destination(route), options)
+        if 'queue' not in options:
+            options = lpmerge(self.expand_destination(
+                              self.app.conf.CELERY_DEFAULT_QUEUE), options)
+        return options
+
+    def expand_destination(self, route):
+        # Route can be a queue name: convenient for direct exchanges.
+        if isinstance(route, basestring):
+            queue, route = route, {}
+        else:
+            # can use defaults from configured queue, but override specific
+            # things (like the routing_key): great for topic exchanges.
+            queue = route.pop('queue', None)
+
+        if queue:
+            try:
+                Q = self.queues[queue]  # noqa
+            except KeyError:
+                if not self.create_missing:
+                    raise QueueNotFound(
+                        'Queue %r is not defined in CELERY_QUEUES' % queue)
+                for key in 'exchange', 'routing_key':
+                    if route.get(key) is None:
+                        route[key] = queue
+                Q = self.app.amqp.queues.add(queue, **route)
+            # needs to be declared by publisher
+            route['queue'] = Q
+        return route
+
+    def lookup_route(self, task, args=None, kwargs=None):
+        return _first_route(self.routes, task, args, kwargs)
+
+
+def prepare(routes):
+    """Expands the :setting:`CELERY_ROUTES` setting."""
+
+    def expand_route(route):
+        if isinstance(route, dict):
+            return MapRoute(route)
+        if isinstance(route, basestring):
+            return mpromise(instantiate, route)
+        return route
+
+    if routes is None:
+        return ()
+    if not isinstance(routes, (list, tuple)):
+        routes = (routes, )
+    return [expand_route(route) for route in routes]

awx/lib/site-packages/celery/app/task.py

@@ -0,0 +1,795 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.task
+    ~~~~~~~~~~~~~~~
+
+    Task Implementation: Task request context, and the base task class.
+
+"""
+from __future__ import absolute_import
+from __future__ import with_statement
+
+from celery import current_app
+from celery import states
+from celery.__compat__ import class_property
+from celery._state import get_current_worker_task, _task_stack
+from celery.canvas import subtask
+from celery.datastructures import ExceptionInfo
+from celery.exceptions import MaxRetriesExceededError, RetryTaskError
+from celery.result import EagerResult
+from celery.utils import gen_task_name, fun_takes_kwargs, uuid, maybe_reraise
+from celery.utils.functional import mattrgetter, maybe_list
+from celery.utils.imports import instantiate
+from celery.utils.mail import ErrorMail
+
+from .annotations import resolve_all as resolve_all_annotations
+from .registry import _unpickle_task
+
+#: extracts attributes related to publishing a message from an object.
+extract_exec_options = mattrgetter(
+    'queue', 'routing_key', 'exchange',
+    'immediate', 'mandatory', 'priority', 'expires',
+    'serializer', 'delivery_mode', 'compression',
+)
+
+
+class Context(object):
+    # Default context
+    logfile = None
+    loglevel = None
+    hostname = None
+    id = None
+    args = None
+    kwargs = None
+    retries = 0
+    eta = None
+    expires = None
+    is_eager = False
+    delivery_info = None
+    taskset = None   # compat alias to group
+    group = None
+    chord = None
+    utc = None
+    called_directly = True
+    callbacks = None
+    errbacks = None
+    timeouts = None
+    _children = None   # see property
+    _protected = 0
+
+    def __init__(self, *args, **kwargs):
+        self.update(*args, **kwargs)
+
+    def update(self, *args, **kwargs):
+        self.__dict__.update(*args, **kwargs)
+
+    def clear(self):
+        self.__dict__.clear()
+
+    def get(self, key, default=None):
+        try:
+            return getattr(self, key)
+        except AttributeError:
+            return default
+
+    def __repr__(self):
+        return '<Context: %r>' % (vars(self, ))
+
+    @property
+    def children(self):
+        # children must be an empy list for every thread
+        if self._children is None:
+            self._children = []
+        return self._children
+
+
+class TaskType(type):
+    """Meta class for tasks.
+
+    Automatically registers the task in the task registry, except
+    if the `abstract` attribute is set.
+
+    If no `name` attribute is provided, then no name is automatically
+    set to the name of the module it was defined in, and the class name.
+
+    """
+
+    def __new__(cls, name, bases, attrs):
+        new = super(TaskType, cls).__new__
+        task_module = attrs.get('__module__') or '__main__'
+
+        # - Abstract class: abstract attribute should not be inherited.
+        if attrs.pop('abstract', None) or not attrs.get('autoregister', True):
+            return new(cls, name, bases, attrs)
+
+        # The 'app' attribute is now a property, with the real app located
+        # in the '_app' attribute.  Previously this was a regular attribute,
+        # so we should support classes defining it.
+        _app1, _app2 = attrs.pop('_app', None), attrs.pop('app', None)
+        app = attrs['_app'] = _app1 or _app2 or current_app
+
+        # - Automatically generate missing/empty name.
+        task_name = attrs.get('name')
+        if not task_name:
+            attrs['name'] = task_name = gen_task_name(app, name, task_module)
+
+        # - Create and register class.
+        # Because of the way import happens (recursively)
+        # we may or may not be the first time the task tries to register
+        # with the framework.  There should only be one class for each task
+        # name, so we always return the registered version.
+        tasks = app._tasks
+        if task_name not in tasks:
+            tasks.register(new(cls, name, bases, attrs))
+        instance = tasks[task_name]
+        instance.bind(app)
+        return instance.__class__
+
+    def __repr__(cls):
+        if cls._app:
+            return '<class %s of %s>' % (cls.__name__, cls._app, )
+        if cls.__v2_compat__:
+            return '<unbound %s (v2 compatible)>' % (cls.__name__, )
+        return '<unbound %s>' % (cls.__name__, )
+
+
+class Task(object):
+    """Task base class.
+
+    When called tasks apply the :meth:`run` method.  This method must
+    be defined by all tasks (that is unless the :meth:`__call__` method
+    is overridden).
+
+    """
+    __metaclass__ = TaskType
+    __trace__ = None
+    __v2_compat__ = False  # set by old base in celery.task.base
+
+    ErrorMail = ErrorMail
+    MaxRetriesExceededError = MaxRetriesExceededError
+
+    #: Execution strategy used, or the qualified name of one.
+    Strategy = 'celery.worker.strategy:default'
+
+    #: This is the instance bound to if the task is a method of a class.
+    __self__ = None
+
+    #: The application instance associated with this task class.
+    _app = None
+
+    #: Name of the task.
+    name = None
+
+    #: If :const:`True` the task is an abstract base class.
+    abstract = True
+
+    #: If disabled the worker will not forward magic keyword arguments.
+    #: Deprecated and scheduled for removal in v4.0.
+    accept_magic_kwargs = False
+
+    #: Maximum number of retries before giving up.  If set to :const:`None`,
+    #: it will **never** stop retrying.
+    max_retries = 3
+
+    #: Default time in seconds before a retry of the task should be
+    #: executed.  3 minutes by default.
+    default_retry_delay = 3 * 60
+
+    #: Rate limit for this task type.  Examples: :const:`None` (no rate
+    #: limit), `'100/s'` (hundred tasks a second), `'100/m'` (hundred tasks
+    #: a minute),`'100/h'` (hundred tasks an hour)
+    rate_limit = None
+
+    #: If enabled the worker will not store task state and return values
+    #: for this task.  Defaults to the :setting:`CELERY_IGNORE_RESULT`
+    #: setting.
+    ignore_result = None
+
+    #: When enabled errors will be stored even if the task is otherwise
+    #: configured to ignore results.
+    store_errors_even_if_ignored = None
+
+    #: If enabled an email will be sent to :setting:`ADMINS` whenever a task
+    #: of this type fails.
+    send_error_emails = None
+
+    #: The name of a serializer that are registered with
+    #: :mod:`kombu.serialization.registry`.  Default is `'pickle'`.
+    serializer = None
+
+    #: Hard time limit.
+    #: Defaults to the :setting:`CELERY_TASK_TIME_LIMIT` setting.
+    time_limit = None
+
+    #: Soft time limit.
+    #: Defaults to the :setting:`CELERY_TASK_SOFT_TIME_LIMIT` setting.
+    soft_time_limit = None
+
+    #: The result store backend used for this task.
+    backend = None
+
+    #: If disabled this task won't be registered automatically.
+    autoregister = True
+
+    #: If enabled the task will report its status as 'started' when the task
+    #: is executed by a worker.  Disabled by default as the normal behaviour
+    #: is to not report that level of granularity.  Tasks are either pending,
+    #: finished, or waiting to be retried.
+    #:
+    #: Having a 'started' status can be useful for when there are long
+    #: running tasks and there is a need to report which task is currently
+    #: running.
+    #:
+    #: The application default can be overridden using the
+    #: :setting:`CELERY_TRACK_STARTED` setting.
+    track_started = None
+
+    #: When enabled messages for this task will be acknowledged **after**
+    #: the task has been executed, and not *just before* which is the
+    #: default behavior.
+    #:
+    #: Please note that this means the task may be executed twice if the
+    #: worker crashes mid execution (which may be acceptable for some
+    #: applications).
+    #:
+    #: The application default can be overridden with the
+    #: :setting:`CELERY_ACKS_LATE` setting.
+    acks_late = None
+
+    #: Default task expiry time.
+    expires = None
+
+    #: Some may expect a request to exist even if the task has not been
+    #: called.  This should probably be deprecated.
+    _default_request = None
+
+    __bound__ = False
+
+    from_config = (
+        ('send_error_emails', 'CELERY_SEND_TASK_ERROR_EMAILS'),
+        ('serializer', 'CELERY_TASK_SERIALIZER'),
+        ('rate_limit', 'CELERY_DEFAULT_RATE_LIMIT'),
+        ('track_started', 'CELERY_TRACK_STARTED'),
+        ('acks_late', 'CELERY_ACKS_LATE'),
+        ('ignore_result', 'CELERY_IGNORE_RESULT'),
+        ('store_errors_even_if_ignored',
+            'CELERY_STORE_ERRORS_EVEN_IF_IGNORED'),
+    )
+
+    __bound__ = False
+
+    # - Tasks are lazily bound, so that configuration is not set
+    # - until the task is actually used
+
+    @classmethod
+    def bind(self, app):
+        was_bound, self.__bound__ = self.__bound__, True
+        self._app = app
+        conf = app.conf
+
+        for attr_name, config_name in self.from_config:
+            if getattr(self, attr_name, None) is None:
+                setattr(self, attr_name, conf[config_name])
+        if self.accept_magic_kwargs is None:
+            self.accept_magic_kwargs = app.accept_magic_kwargs
+        if self.backend is None:
+            self.backend = app.backend
+
+        # decorate with annotations from config.
+        if not was_bound:
+            self.annotate()
+
+            from celery.utils.threads import LocalStack
+            self.request_stack = LocalStack()
+
+        # PeriodicTask uses this to add itself to the PeriodicTask schedule.
+        self.on_bound(app)
+
+        return app
+
+    @classmethod
+    def on_bound(self, app):
+        """This method can be defined to do additional actions when the
+        task class is bound to an app."""
+        pass
+
+    @classmethod
+    def _get_app(self):
+        if not self.__bound__ or self._app is None:
+            # The app property's __set__  method is not called
+            # if Task.app is set (on the class), so must bind on use.
+            self.bind(current_app)
+        return self._app
+    app = class_property(_get_app, bind)
+
+    @classmethod
+    def annotate(self):
+        for d in resolve_all_annotations(self.app.annotations, self):
+            for key, value in d.iteritems():
+                if key.startswith('@'):
+                    self.add_around(key[1:], value)
+                else:
+                    setattr(self, key, value)
+
+    @classmethod
+    def add_around(self, attr, around):
+        orig = getattr(self, attr)
+        if getattr(orig, '__wrapped__', None):
+            orig = orig.__wrapped__
+        meth = around(orig)
+        meth.__wrapped__ = orig
+        setattr(self, attr, meth)
+
+    def __call__(self, *args, **kwargs):
+        _task_stack.push(self)
+        self.push_request()
+        try:
+            # add self if this is a bound task
+            if self.__self__ is not None:
+                return self.run(self.__self__, *args, **kwargs)
+            return self.run(*args, **kwargs)
+        finally:
+            self.pop_request()
+            _task_stack.pop()
+
+    # - tasks are pickled into the name of the task only, and the reciever
+    # - simply grabs it from the local registry.
+    def __reduce__(self):
+        return (_unpickle_task, (self.name, ), None)
+
+    def run(self, *args, **kwargs):
+        """The body of the task executed by workers."""
+        raise NotImplementedError('Tasks must define the run method.')
+
+    def start_strategy(self, app, consumer):
+        return instantiate(self.Strategy, self, app, consumer)
+
+    def delay(self, *args, **kwargs):
+        """Star argument version of :meth:`apply_async`.
+
+        Does not support the extra options enabled by :meth:`apply_async`.
+
+        :param \*args: positional arguments passed on to the task.
+        :param \*\*kwargs: keyword arguments passed on to the task.
+
+        :returns :class:`celery.result.AsyncResult`:
+
+        """
+        return self.apply_async(args, kwargs)
+
+    def apply_async(self, args=None, kwargs=None,
+                    task_id=None, producer=None, connection=None, router=None,
+                    link=None, link_error=None, publisher=None,
+                    add_to_parent=True, **options):
+        """Apply tasks asynchronously by sending a message.
+
+        :keyword args: The positional arguments to pass on to the
+                       task (a :class:`list` or :class:`tuple`).
+
+        :keyword kwargs: The keyword arguments to pass on to the
+                         task (a :class:`dict`)
+
+        :keyword countdown: Number of seconds into the future that the
+                            task should execute. Defaults to immediate
+                            execution (do not confuse with the
+                            `immediate` flag, as they are unrelated).
+
+        :keyword eta: A :class:`~datetime.datetime` object describing
+                      the absolute time and date of when the task should
+                      be executed.  May not be specified if `countdown`
+                      is also supplied.  (Do not confuse this with the
+                      `immediate` flag, as they are unrelated).
+
+        :keyword expires: Either a :class:`int`, describing the number of
+                          seconds, or a :class:`~datetime.datetime` object
+                          that describes the absolute time and date of when
+                          the task should expire.  The task will not be
+                          executed after the expiration time.
+
+        :keyword connection: Re-use existing broker connection instead
+                             of establishing a new one.
+
+        :keyword retry: If enabled sending of the task message will be retried
+                        in the event of connection loss or failure.  Default
+                        is taken from the :setting:`CELERY_TASK_PUBLISH_RETRY`
+                        setting.  Note you need to handle the
+                        producer/connection manually for this to work.
+
+        :keyword retry_policy:  Override the retry policy used.  See the
+                                :setting:`CELERY_TASK_PUBLISH_RETRY` setting.
+
+        :keyword routing_key: Custom routing key used to route the task to a
+                              worker server. If in combination with a
+                              ``queue`` argument only used to specify custom
+                              routing keys to topic exchanges.
+
+        :keyword queue: The queue to route the task to.  This must be a key
+                        present in :setting:`CELERY_QUEUES`, or
+                        :setting:`CELERY_CREATE_MISSING_QUEUES` must be
+                        enabled.  See :ref:`guide-routing` for more
+                        information.
+
+        :keyword exchange: Named custom exchange to send the task to.
+                           Usually not used in combination with the ``queue``
+                           argument.
+
+        :keyword priority: The task priority, a number between 0 and 9.
+                           Defaults to the :attr:`priority` attribute.
+
+        :keyword serializer: A string identifying the default
+                             serialization method to use.  Can be `pickle`,
+                             `json`, `yaml`, `msgpack` or any custom
+                             serialization method that has been registered
+                             with :mod:`kombu.serialization.registry`.
+                             Defaults to the :attr:`serializer` attribute.
+
+        :keyword compression: A string identifying the compression method
+                              to use.  Can be one of ``zlib``, ``bzip2``,
+                              or any custom compression methods registered with
+                              :func:`kombu.compression.register`. Defaults to
+                              the :setting:`CELERY_MESSAGE_COMPRESSION`
+                              setting.
+        :keyword link: A single, or a list of subtasks to apply if the
+                       task exits successfully.
+        :keyword link_error: A single, or a list of subtasks to apply
+                      if an error occurs while executing the task.
+
+        :keyword producer: :class:~@amqp.TaskProducer` instance to use.
+        :keyword add_to_parent: If set to True (default) and the task
+            is applied while executing another task, then the result
+            will be appended to the parent tasks ``request.children``
+            attribute.
+        :keyword publisher: Deprecated alias to ``producer``.
+
+        Also supports all keyword arguments supported by
+        :meth:`kombu.messaging.Producer.publish`.
+
+        .. note::
+            If the :setting:`CELERY_ALWAYS_EAGER` setting is set, it will
+            be replaced by a local :func:`apply` call instead.
+
+        """
+        producer = producer or publisher
+        app = self._get_app()
+        router = router or self.app.amqp.router
+        conf = app.conf
+
+        # add 'self' if this is a bound method.
+        if self.__self__ is not None:
+            args = (self.__self__, ) + tuple(args)
+
+        if conf.CELERY_ALWAYS_EAGER:
+            return self.apply(args, kwargs, task_id=task_id, **options)
+        options = dict(extract_exec_options(self), **options)
+        options = router.route(options, self.name, args, kwargs)
+
+        if connection:
+            producer = app.amqp.TaskProducer(connection)
+        with app.producer_or_acquire(producer) as P:
+            task_id = P.publish_task(self.name, args, kwargs,
+                                     task_id=task_id,
+                                     callbacks=maybe_list(link),
+                                     errbacks=maybe_list(link_error),
+                                     **options)
+        result = self.AsyncResult(task_id)
+        if add_to_parent:
+            parent = get_current_worker_task()
+            if parent:
+                parent.request.children.append(result)
+        return result
+
+    def subtask_from_request(self, request=None, args=None, kwargs=None,
+                             **extra_options):
+
+        request = self.request if request is None else request
+        args = request.args if args is None else args
+        kwargs = request.kwargs if kwargs is None else kwargs
+        delivery_info = request.delivery_info or {}
+        options = {
+            'task_id': request.id,
+            'link': request.callbacks,
+            'link_error': request.errbacks,
+            'exchange': delivery_info.get('exchange'),
+            'routing_key': delivery_info.get('routing_key')
+        }
+        return self.subtask(args, kwargs, options, type=self, **extra_options)
+
+    def retry(self, args=None, kwargs=None, exc=None, throw=True,
+              eta=None, countdown=None, max_retries=None, **options):
+        """Retry the task.
+
+        :param args: Positional arguments to retry with.
+        :param kwargs: Keyword arguments to retry with.
+        :keyword exc: Custom exception to report when the max restart
+            limit has been exceeded (default:
+            :exc:`~celery.exceptions.MaxRetriesExceededError`).
+
+            If this argument is set and retry is called while
+            an exception was raised (``sys.exc_info()`` is set)
+            it will attempt to reraise the current exception.
+
+            If no exception was raised it will raise the ``exc``
+            argument provided.
+        :keyword countdown: Time in seconds to delay the retry for.
+        :keyword eta: Explicit time and date to run the retry at
+                      (must be a :class:`~datetime.datetime` instance).
+        :keyword max_retries: If set, overrides the default retry limit.
+        :keyword \*\*options: Any extra options to pass on to
+                              meth:`apply_async`.
+        :keyword throw: If this is :const:`False`, do not raise the
+                        :exc:`~celery.exceptions.RetryTaskError` exception,
+                        that tells the worker to mark the task as being
+                        retried.  Note that this means the task will be
+                        marked as failed if the task raises an exception,
+                        or successful if it returns.
+
+        :raises celery.exceptions.RetryTaskError: To tell the worker that
+            the task has been re-sent for retry. This always happens,
+            unless the `throw` keyword argument has been explicitly set
+            to :const:`False`, and is considered normal operation.
+
+        **Example**
+
+        .. code-block:: python
+
+            >>> @task()
+            >>> def tweet(auth, message):
+            ...     twitter = Twitter(oauth=auth)
+            ...     try:
+            ...         twitter.post_status_update(message)
+            ...     except twitter.FailWhale, exc:
+            ...         # Retry in 5 minutes.
+            ...         raise tweet.retry(countdown=60 * 5, exc=exc)
+
+        Although the task will never return above as `retry` raises an
+        exception to notify the worker, we use `return` in front of the retry
+        to convey that the rest of the block will not be executed.
+
+        """
+        request = self.request
+        retries = request.retries + 1
+        max_retries = self.max_retries if max_retries is None else max_retries
+
+        # Not in worker or emulated by (apply/always_eager),
+        # so just raise the original exception.
+        if request.called_directly:
+            maybe_reraise()  # raise orig stack if PyErr_Occurred
+            raise exc or RetryTaskError('Task can be retried', None)
+
+        if not eta and countdown is None:
+            countdown = self.default_retry_delay
+
+        S = self.subtask_from_request(
+            request, args, kwargs,
+            countdown=countdown, eta=eta, retries=retries,
+            **options
+        )
+
+        if max_retries is not None and retries > max_retries:
+            if exc:
+                maybe_reraise()
+            raise self.MaxRetriesExceededError(
+                """Can't retry %s[%s] args:%s kwargs:%s""" % (
+                    self.name, request.id, S.args, S.kwargs))
+
+        # If task was executed eagerly using apply(),
+        # then the retry must also be executed eagerly.
+        S.apply().get() if request.is_eager else S.apply_async()
+        ret = RetryTaskError(exc=exc, when=eta or countdown)
+        if throw:
+            raise ret
+        return ret
+
+    def apply(self, args=None, kwargs=None, **options):
+        """Execute this task locally, by blocking until the task returns.
+
+        :param args: positional arguments passed on to the task.
+        :param kwargs: keyword arguments passed on to the task.
+        :keyword throw: Re-raise task exceptions.  Defaults to
+                        the :setting:`CELERY_EAGER_PROPAGATES_EXCEPTIONS`
+                        setting.
+
+        :rtype :class:`celery.result.EagerResult`:
+
+        """
+        # trace imports Task, so need to import inline.
+        from celery.task.trace import eager_trace_task
+
+        app = self._get_app()
+        args = args or ()
+        # add 'self' if this is a bound method.
+        if self.__self__ is not None:
+            args = (self.__self__, ) + tuple(args)
+        kwargs = kwargs or {}
+        task_id = options.get('task_id') or uuid()
+        retries = options.get('retries', 0)
+        throw = app.either('CELERY_EAGER_PROPAGATES_EXCEPTIONS',
+                           options.pop('throw', None))
+
+        # Make sure we get the task instance, not class.
+        task = app._tasks[self.name]
+
+        request = {'id': task_id,
+                   'retries': retries,
+                   'is_eager': True,
+                   'logfile': options.get('logfile'),
+                   'loglevel': options.get('loglevel', 0),
+                   'delivery_info': {'is_eager': True}}
+        if self.accept_magic_kwargs:
+            default_kwargs = {'task_name': task.name,
+                              'task_id': task_id,
+                              'task_retries': retries,
+                              'task_is_eager': True,
+                              'logfile': options.get('logfile'),
+                              'loglevel': options.get('loglevel', 0),
+                              'delivery_info': {'is_eager': True}}
+            supported_keys = fun_takes_kwargs(task.run, default_kwargs)
+            extend_with = dict((key, val)
+                               for key, val in default_kwargs.items()
+                               if key in supported_keys)
+            kwargs.update(extend_with)
+
+        tb = None
+        retval, info = eager_trace_task(task, task_id, args, kwargs,
+                                        request=request, propagate=throw)
+        if isinstance(retval, ExceptionInfo):
+            retval, tb = retval.exception, retval.traceback
+        state = states.SUCCESS if info is None else info.state
+        return EagerResult(task_id, retval, state, traceback=tb)
+
+    def AsyncResult(self, task_id, **kwargs):
+        """Get AsyncResult instance for this kind of task.
+
+        :param task_id: Task id to get result for.
+
+        """
+        return self._get_app().AsyncResult(task_id, backend=self.backend,
+                                           task_name=self.name, **kwargs)
+
+    def subtask(self, args=None, *starargs, **starkwargs):
+        """Returns :class:`~celery.subtask` object for
+        this task, wrapping arguments and execution options
+        for a single task invocation."""
+        return subtask(self, args, *starargs, **starkwargs)
+
+    def s(self, *args, **kwargs):
+        """``.s(*a, **k) -> .subtask(a, k)``"""
+        return self.subtask(args, kwargs)
+
+    def si(self, *args, **kwargs):
+        """``.si(*a, **k) -> .subtask(a, k, immutable=True)``"""
+        return self.subtask(args, kwargs, immutable=True)
+
+    def chunks(self, it, n):
+        """Creates a :class:`~celery.canvas.chunks` task for this task."""
+        from celery import chunks
+        return chunks(self.s(), it, n)
+
+    def map(self, it):
+        """Creates a :class:`~celery.canvas.xmap` task from ``it``."""
+        from celery import xmap
+        return xmap(self.s(), it)
+
+    def starmap(self, it):
+        """Creates a :class:`~celery.canvas.xstarmap` task from ``it``."""
+        from celery import xstarmap
+        return xstarmap(self.s(), it)
+
+    def update_state(self, task_id=None, state=None, meta=None):
+        """Update task state.
+
+        :keyword task_id: Id of the task to update, defaults to the
+                          id of the current task
+        :keyword state: New state (:class:`str`).
+        :keyword meta: State metadata (:class:`dict`).
+
+
+
+        """
+        if task_id is None:
+            task_id = self.request.id
+        self.backend.store_result(task_id, meta, state)
+
+    def on_success(self, retval, task_id, args, kwargs):
+        """Success handler.
+
+        Run by the worker if the task executes successfully.
+
+        :param retval: The return value of the task.
+        :param task_id: Unique id of the executed task.
+        :param args: Original arguments for the executed task.
+        :param kwargs: Original keyword arguments for the executed task.
+
+        The return value of this handler is ignored.
+
+        """
+        pass
+
+    def on_retry(self, exc, task_id, args, kwargs, einfo):
+        """Retry handler.
+
+        This is run by the worker when the task is to be retried.
+
+        :param exc: The exception sent to :meth:`retry`.
+        :param task_id: Unique id of the retried task.
+        :param args: Original arguments for the retried task.
+        :param kwargs: Original keyword arguments for the retried task.
+
+        :keyword einfo: :class:`~celery.datastructures.ExceptionInfo`
+                        instance, containing the traceback.
+
+        The return value of this handler is ignored.
+
+        """
+        pass
+
+    def on_failure(self, exc, task_id, args, kwargs, einfo):
+        """Error handler.
+
+        This is run by the worker when the task fails.
+
+        :param exc: The exception raised by the task.
+        :param task_id: Unique id of the failed task.
+        :param args: Original arguments for the task that failed.
+        :param kwargs: Original keyword arguments for the task
+                       that failed.
+
+        :keyword einfo: :class:`~celery.datastructures.ExceptionInfo`
+                        instance, containing the traceback.
+
+        The return value of this handler is ignored.
+
+        """
+        pass
+
+    def after_return(self, status, retval, task_id, args, kwargs, einfo):
+        """Handler called after the task returns.
+
+        :param status: Current task state.
+        :param retval: Task return value/exception.
+        :param task_id: Unique id of the task.
+        :param args: Original arguments for the task that failed.
+        :param kwargs: Original keyword arguments for the task
+                       that failed.
+
+        :keyword einfo: :class:`~celery.datastructures.ExceptionInfo`
+                        instance, containing the traceback (if any).
+
+        The return value of this handler is ignored.
+
+        """
+        pass
+
+    def send_error_email(self, context, exc, **kwargs):
+        if self.send_error_emails and \
+                not getattr(self, 'disable_error_emails', None):
+            self.ErrorMail(self, **kwargs).send(context, exc)
+
+    def push_request(self, *args, **kwargs):
+        self.request_stack.push(Context(*args, **kwargs))
+
+    def pop_request(self):
+        self.request_stack.pop()
+
+    def __repr__(self):
+        """`repr(task)`"""
+        if self.__self__:
+            return '<bound task %s of %r>' % (self.name, self.__self__)
+        return '<@task: %s>' % (self.name, )
+
+    def _get_request(self):
+        """Get current request object."""
+        req = self.request_stack.top
+        if req is None:
+            # task was not called, but some may still expect a request
+            # to be there, perhaps that should be deprecated.
+            if self._default_request is None:
+                self._default_request = Context()
+            return self._default_request
+        return req
+    request = property(_get_request)
+
+    @property
+    def __name__(self):
+        return self.__class__.__name__
+BaseTask = Task  # compat alias

awx/lib/site-packages/celery/app/utils.py

@@ -0,0 +1,178 @@
+# -*- coding: utf-8 -*-
+"""
+    celery.app.utils
+    ~~~~~~~~~~~~~~~~
+
+    App utilities: Compat settings, bugreport tool, pickling apps.
+
+"""
+from __future__ import absolute_import
+
+import os
+import platform as _platform
+import re
+
+from celery import platforms
+from celery.datastructures import ConfigurationView
+from celery.utils.text import pretty
+from celery.utils.imports import qualname
+
+from .defaults import find
+
+#: Format used to generate bugreport information.
+BUGREPORT_INFO = """
+software -> celery:%(celery_v)s kombu:%(kombu_v)s py:%(py_v)s
+            billiard:%(billiard_v)s %(driver_v)s
+platform -> system:%(system)s arch:%(arch)s imp:%(py_i)s
+loader   -> %(loader)s
+settings -> transport:%(transport)s results:%(results)s
+
+%(human_settings)s
+"""
+
+HIDDEN_SETTINGS = re.compile(
+    'API|TOKEN|KEY|SECRET|PASS|PROFANITIES_LIST|SIGNATURE|DATABASE',
+    re.IGNORECASE,
+)
+
+
+class Settings(ConfigurationView):
+    """Celery settings object."""
+
+    @property
+    def CELERY_RESULT_BACKEND(self):
+        return self.first('CELERY_RESULT_BACKEND', 'CELERY_BACKEND')
+
+    @property
+    def BROKER_TRANSPORT(self):
+        return self.first('BROKER_TRANSPORT',
+                          'BROKER_BACKEND', 'CARROT_BACKEND')
+
+    @property
+    def BROKER_BACKEND(self):
+        """Deprecated compat alias to :attr:`BROKER_TRANSPORT`."""
+        return self.BROKER_TRANSPORT
+
+    @property
+    def BROKER_HOST(self):
+        return (os.environ.get('CELERY_BROKER_URL') or
+                self.first('BROKER_URL', 'BROKER_HOST'))
+
+    @property
+    def CELERY_TIMEZONE(self):
+        # this way we also support django's time zone.
+        return self.first('CELERY_TIMEZONE', 'TIME_ZONE')
+
+    def without_defaults(self):
+        """Returns the current configuration, but without defaults."""
+        # the last stash is the default settings, so just skip that
+        return Settings({}, self._order[:-1])
+
+    def find_option(self, name, namespace='celery'):
+        """Search for option by name.
+
+        Will return ``(namespace, option_name, Option)`` tuple, e.g.::
+
+            >>> celery.conf.find_option('disable_rate_limits')
+            ('CELERY', 'DISABLE_RATE_LIMITS',
+             <Option: type->bool default->False>))
+
+        :param name: Name of option, cannot be partial.
+        :keyword namespace: Preferred namespace (``CELERY`` by default).
+
+        """
+        return find(name, namespace)
+
+    def find_value_for_key(self, name, namespace='celery'):
+        """Shortcut to ``get_by_parts(*find_option(name)[:-1])``"""
+        return self.get_by_parts(*self.find_option(name, namespace)[:-1])
+
+    def get_by_parts(self, *parts):
+        """Returns the current value for setting specified as a path.
+
+        Example::
+
+            >>> celery.conf.get_by_parts('CELERY', 'DISABLE_RATE_LIMITS')
+            False
+
+        """
+        return self['_'.join(part for part in parts if part)]
+
+    def humanize(self):
+        """Returns a human readable string showing changes to the
+        configuration."""
+        return '\n'.join(
+            '%s: %s' % (key, pretty(value, width=50))
+            for key, value in filter_hidden_settings(dict(
+                (k, v) for k, v in self.without_defaults().iteritems()
+                if k.isupper() and not k.startswith('_'))).iteritems())
+
+
+class AppPickler(object):
+    """Default application pickler/unpickler."""
+
+    def __call__(self, cls, *args):
+        kwargs = self.build_kwargs(*args)
+        app = self.construct(cls, **kwargs)
+        self.prepare(app, **kwargs)
+        return app
+
+    def prepare(self, app, **kwargs):
+        app.conf.update(kwargs['changes'])
+
+    def build_kwargs(self, *args):
+        return self.build_standard_kwargs(*args)
+
+    def build_standard_kwargs(self, main, changes, loader, backend, amqp,
+                              events, log, control, accept_magic_kwargs,
+                              config_source=None):
+        return dict(main=main, loader=loader, backend=backend, amqp=amqp,
+                    changes=changes, events=events, log=log, control=control,
+                    set_as_current=False,
+                    accept_magic_kwargs=accept_magic_kwargs,
+                    config_source=config_source)
+
+    def construct(self, cls, **kwargs):
+        return cls(**kwargs)
+
+
+def _unpickle_app(cls, pickler, *args):
+    return pickler()(cls, *args)
+
+
+def filter_hidden_settings(conf):
+
+    def maybe_censor(key, value):
+        return '********' if HIDDEN_SETTINGS.search(key) else value
+
+    return dict((k, maybe_censor(k, v)) for k, v in conf.iteritems())
+
+
+def bugreport(app):
+    """Returns a string containing information useful in bug reports."""
+    import billiard
+    import celery
+    import kombu
+
+    try:
+        conn = app.connection()
+        driver_v = '%s:%s' % (conn.transport.driver_name,
+                              conn.transport.driver_version())
+        transport = conn.transport_cls
+    except Exception:
+        transport = driver_v = ''
+
+    return BUGREPORT_INFO % {
+        'system': _platform.system(),
+        'arch': ', '.join(p for p in _platform.architecture() if p),
+        'py_i': platforms.pyimplementation(),
+        'celery_v': celery.VERSION_BANNER,
+        'kombu_v': kombu.__version__,
+        'billiard_v': billiard.__version__,
+        'py_v': _platform.python_version(),
+        'driver_v': driver_v,
+        'transport': transport,
+        'results': app.conf.CELERY_RESULT_BACKEND or 'disabled',
+        'human_settings': app.conf.humanize(),
+        'loader': qualname(app.loader.__class__),
+    }