txzookeeper-0.9.8/ 0000775 0001750 0001750 00000000000 12144725212 014333 5 ustar kapil kapil 0000000 0000000 txzookeeper-0.9.8/txzookeeper.egg-info/ 0000775 0001750 0001750 00000000000 12144725212 020404 5 ustar kapil kapil 0000000 0000000 txzookeeper-0.9.8/txzookeeper.egg-info/top_level.txt 0000664 0001750 0001750 00000000014 12144725212 023131 0 ustar kapil kapil 0000000 0000000 txzookeeper
txzookeeper-0.9.8/txzookeeper.egg-info/PKG-INFO 0000664 0001750 0001750 00000001262 12144725212 021502 0 ustar kapil kapil 0000000 0000000 Metadata-Version: 1.1
Name: txzookeeper
Version: 0.9.8
Summary: Twisted api for Apache Zookeeper
Home-page: https://launchpad.net/txzookeeper
Author: Juju Developers
Author-email: juju@lists.ubuntu.com
License: LGPL
Description:
Twisted API for Apache Zookeeper. Includes a distributed lock, and several
queue implementations.
Platform: UNKNOWN
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: System Administrators
Classifier: Intended Audience :: Information Technology
Classifier: Programming Language :: Python
Classifier: Topic :: Database
Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)
txzookeeper-0.9.8/txzookeeper.egg-info/dependency_links.txt 0000664 0001750 0001750 00000000001 12144725212 024452 0 ustar kapil kapil 0000000 0000000
txzookeeper-0.9.8/txzookeeper.egg-info/SOURCES.txt 0000664 0001750 0001750 00000001431 12144725212 022267 0 ustar kapil kapil 0000000 0000000 setup.py
txzookeeper/__init__.py
txzookeeper/client.py
txzookeeper/lock.py
txzookeeper/managed.py
txzookeeper/node.py
txzookeeper/queue.py
txzookeeper/retry.py
txzookeeper/utils.py
txzookeeper.egg-info/PKG-INFO
txzookeeper.egg-info/SOURCES.txt
txzookeeper.egg-info/dependency_links.txt
txzookeeper.egg-info/top_level.txt
txzookeeper/tests/__init__.py
txzookeeper/tests/common.py
txzookeeper/tests/mocker.py
txzookeeper/tests/proxy.py
txzookeeper/tests/test_client.py
txzookeeper/tests/test_conn_failure.py
txzookeeper/tests/test_lock.py
txzookeeper/tests/test_managed.py
txzookeeper/tests/test_node.py
txzookeeper/tests/test_queue.py
txzookeeper/tests/test_retry.py
txzookeeper/tests/test_security.py
txzookeeper/tests/test_session.py
txzookeeper/tests/test_utils.py
txzookeeper/tests/utils.py txzookeeper-0.9.8/txzookeeper/ 0000775 0001750 0001750 00000000000 12144725212 016712 5 ustar kapil kapil 0000000 0000000 txzookeeper-0.9.8/txzookeeper/queue.py 0000664 0001750 0001750 00000040274 11745322401 020416 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
"""
Several distributed multiprocess queue implementations.
The C{Queue} implementation follows closely the apache zookeeper recipe, it
provides no guarantees beyond isolation and concurrency of retrieval of items.
The C{ReliableQueue} implementation, provides isolation, and concurrency, as
well guarantees that if a consumer dies before processing an item, that item is
made available to another consumer.
The C{SerializedQueue} implementation provides for strict in order processing
of items within a queue.
"""
import zookeeper
from twisted.internet.defer import Deferred, fail
from twisted.python.failure import Failure
from txzookeeper.lock import Lock
from txzookeeper.client import ZOO_OPEN_ACL_UNSAFE
class Queue(object):
"""
Implementation is based off the apache zookeeper Queue recipe.
There are some things to keep in mind when using this queue implementation.
Its primarily to enforce isolation and concurrent access, however it does
not provide for reliable consumption. An error condition in a queue
consumer must requeue the item, else its lost, as its removed from
zookeeper on retrieval in this implementation. This implementation more
closely mirrors the behavior and api of the pythonstandard library Queue,
or multiprocessing.Queue ableit with the caveat of only strings for queue
items.
"""
prefix = "entry-"
def __init__(self, path, client, acl=None, persistent=False):
"""
@param client: A connected C{ZookeeperClient} instance.
@param path: The path to the queue inthe zookeeper hierarchy.
@param acl: An acl to be used for queue items.
@param persistent: Boolean flag which denotes if items in the queue are
persistent.
"""
self._path = path
self._client = client
self._persistent = persistent
if acl is None:
acl = [ZOO_OPEN_ACL_UNSAFE]
self._acl = acl
@property
def path(self):
"""Path to the queue."""
return self._path
@property
def persistent(self):
"""If the queue is persistent returns True."""
return self._persistent
def get(self):
"""
Get and remove an item from the queue. If no item is available
at the moment, a deferred is return that will fire when an item
is available.
"""
def on_queue_items_changed(*args):
"""Event watcher on queue node child events."""
if request.complete or not self._client.connected:
return # pragma: no cover
if request.processing_children:
# If deferred stack is currently processing a set of children
# defer refetching the children till its done.
request.refetch_children = True
else:
# Else the item get request is just waiting for a watch,
# restart the get.
self._get(request)
request = GetRequest(Deferred(), on_queue_items_changed)
self._get(request)
return request.deferred
def put(self, item):
"""
Put an item into the queue.
@param item: String data to be put on the queue.
"""
if not isinstance(item, str):
return fail(ValueError("queue items must be strings"))
flags = zookeeper.SEQUENCE
if not self._persistent:
flags = flags | zookeeper.EPHEMERAL
d = self._client.create(
"/".join((self._path, self.prefix)), item, self._acl, flags)
return d
def qsize(self):
"""
Return the approximate size of the queue. This value is always
effectively a snapshot. Returns a deferred returning an integer.
"""
d = self._client.exists(self._path)
def on_success(stat):
return stat["numChildren"]
d.addCallback(on_success)
return d
def _get(self, request):
request.processing_children = True
d, w = self._client.get_children_and_watch(self._path)
w.addCallback(request.child_watcher)
d.addCallback(self._get_item, request)
return d
def _get_item(self, children, request):
def fetch_node(name):
path = "/".join((self._path, name))
d = self._client.get(path)
d.addCallback(on_get_node_success)
d.addErrback(on_no_node)
return d
def on_get_node_success((data, stat)):
d = self._client.delete("/".join((self._path, name)))
d.addCallback(on_delete_node_success, data)
d.addErrback(on_no_node)
return d
def on_delete_node_success(result_code, data):
request.processing_children = False
request.callback(data)
def on_no_node(failure=None):
if failure and not failure.check(zookeeper.NoNodeException):
request.errback(failure)
return
if children:
name = children.pop(0)
return fetch_node(name)
# Refetching deferred until we process all the children from
# from a get children call.
request.processing_children = False
if request.refetch_children:
request.refetch_children = False
return self._get(request)
if not children:
return on_no_node()
children.sort()
name = children.pop(0)
return fetch_node(name)
class GetRequest(object):
"""
An encapsulation of a consumer request to fetch an item from the queue.
@refetch_children - boolean field, when true signals that children should
be refetched after processing the current set of children.
@child_watcher -The queue child/item watcher.
@processing_children - Boolean flag, set to true when the last known
children of the queue are being processed. If a watch fires while the
children are being processed it sets the refetch_children flag to true
instead of getting the children immediately.
@deferred - The deferred representing retrieving an item from the queue.
"""
def __init__(self, deferred, watcher):
self.deferred = deferred
self.child_watcher = watcher
self.processing_children = False
self.refetch_children = False
@property
def complete(self):
return self.deferred.called
def callback(self, data):
self.deferred.callback(data)
def errback(self, error):
self.deferred.errback(error)
class QueueItem(object):
"""
An encapsulation of a work item put into a queue. The work item data is
accessible via the data attribute. When the item has been processed by
the consumer, the delete method can be invoked to remove the item
permanently from the queue.
An optional processed callback maybe passed to the constructor that will
be invoked after the node has been processed.
"""
def __init__(self, path, data, client, processed_callback=None):
self._path = path
self._data = data
self._client = client
self._processed_callback = processed_callback
@property
def data(self):
return self._data
@property
def path(self):
return self._path
def delete(self):
"""
Delete the item node and the item processing node in the queue.
Typically invoked by a queue consumer, to signal succesful processing
of the queue item.
"""
d = self._client.delete(self.path)
if self._processed_callback:
d.addCallback(self._processed_callback, self.path)
return d
class ReliableQueue(Queue):
"""
A distributed queue. It varies from a C{Queue} in that it ensures any
item consumed from the queue is explicitly ack'd by the consumer.
If the consumer dies after retrieving an item before ack'ing the item.
The item will be made available to another consumer. To encapsulate the
acking behavior the queue item data is returned in a C{QueueItem} instance,
with a delete method that will remove it from the queue after processing.
Reliable queues may be persistent or transient. If the queue is durable,
than any item added to the queue must be processed in order to be removed.
If the queue is transient, then any jobs placed in the queue by a client
are removed when the client is closed, regardless of whether the job
has been processed or not.
"""
def _item_processed_callback(self, result_code, item_path):
return self._client.delete(item_path + "-processing")
def _filter_children(self, children, suffix="-processing"):
"""
Filter any children currently being processed, modified in place.
"""
children.sort()
for name in list(children):
# remove any processing nodes and their associated queue item.
if name.endswith(suffix):
children.remove(name)
item_name = name[:-len(suffix)]
if item_name in children:
children.remove(item_name)
def _get_item(self, children, request):
def check_node(name):
"""Check the node still exists."""
path = "/".join((self._path, name))
d = self._client.exists(path)
d.addCallback(on_node_exists, path)
d.addErrback(on_reservation_failed)
return d
def on_node_exists(stat, path):
"""Reserve the node for consumer processing."""
d = self._client.create(path + "-processing",
flags=zookeeper.EPHEMERAL)
d.addCallback(on_reservation_success, path)
d.addErrback(on_reservation_failed)
return d
def on_reservation_success(processing_path, path):
"""Fetch the node data to return"""
d = self._client.get(path)
d.addCallback(on_get_node_success, path)
d.addErrback(on_get_node_failed, path)
return d
def on_get_node_failed(failure, path):
"""If we can't fetch the node, delete the processing node."""
d = self._client.delete(path + "-processing")
# propogate unexpected errors appropriately
if not failure.check(zookeeper.NoNodeException):
d.addCallback(lambda x: request.errback(failure))
else:
d.addCallback(on_reservation_failed)
return d
def on_get_node_success((data, stat), path):
"""If we got the node, we're done."""
request.processing_children = False
request.callback(
QueueItem(
path, data, self._client, self._item_processed_callback))
def on_reservation_failed(failure=None):
"""If we can't get the node or reserve, continue processing
the children."""
if failure and not failure.check(
zookeeper.NodeExistsException, zookeeper.NoNodeException):
request.processing_children = True
request.errback(failure)
return
if children:
name = children.pop(0)
return check_node(name)
# If a watch fired while processing children, process it
# after the children list is exhausted.
request.processing_children = False
if request.refetch_children:
request.refetch_children = False
return self._get(request)
self._filter_children(children)
if not children:
return on_reservation_failed()
name = children.pop(0)
return check_node(name)
class SerializedQueue(Queue):
"""
A serialized queue ensures even with multiple consumers items are retrieved
and processed in the order they where placed in the queue.
This implementation aggregates a reliable queue, with a lock to provide
for serialized consumer access. The lock is released only when a queue item
has been processed.
"""
def __init__(self, path, client, acl=None, persistent=False):
super(SerializedQueue, self).__init__(path, client, acl, persistent)
self._lock = Lock("%s/%s" % (self.path, "_lock"), client)
def _item_processed_callback(self, result_code, item_path):
return self._lock.release()
def _filter_children(self, children, suffix="-processing"):
"""
Filter the lock from consideration as an item to be processed.
"""
children.sort()
for name in list(children):
if name.startswith('_'):
children.remove(name)
def _on_lock_directory_does_not_exist(self, failure):
"""
If the lock directory does not exist, go ahead and create it and
attempt to acquire the lock.
"""
failure.trap(zookeeper.NoNodeException)
d = self._client.create(self._lock.path)
d.addBoth(self._on_lock_created_or_exists)
return d
def _on_lock_created_or_exists(self, failure):
"""
The lock node creation will either result in success or node exists
error, if a concurrent client created the node first. In either case
we proceed with attempting to acquire the lock.
"""
if isinstance(failure, Failure):
failure.trap(zookeeper.NodeExistsException)
d = self._lock.acquire()
return d
def _on_lock_acquired(self, lock):
"""
After the exclusive queue lock is acquired, we proceed with an attempt
to fetch an item from the queue.
"""
d = super(SerializedQueue, self).get()
return d
def get(self):
"""
Get and remove an item from the queue. If no item is available
at the moment, a deferred is return that will fire when an item
is available.
"""
d = self._lock.acquire()
d.addErrback(self._on_lock_directory_does_not_exist)
d.addCallback(self._on_lock_acquired)
return d
def _get_item(self, children, request):
def fetch_node(name):
path = "/".join((self._path, name))
d = self._client.get(path)
d.addCallback(on_node_retrieved, path)
d.addErrback(on_reservation_failed)
return d
def on_node_retrieved((data, stat), path):
request.processing_children = False
request.callback(
QueueItem(
path, data, self._client, self._item_processed_callback))
def on_reservation_failed(failure=None):
"""If we can't get the node or reserve, continue processing
the children."""
if failure and not failure.check(
zookeeper.NodeExistsException, zookeeper.NoNodeException):
request.processing_children = True
request.errback(failure)
return
if children:
name = children.pop(0)
return fetch_node(name)
# If a watch fired while processing children, process it
# after the children list is exhausted.
request.processing_children = False
if request.refetch_children:
request.refetch_children = False
return self._get(request)
self._filter_children(children)
if not children:
return on_reservation_failed()
name = children.pop(0)
return fetch_node(name)
txzookeeper-0.9.8/txzookeeper/managed.py 0000664 0001750 0001750 00000032427 12144707107 020673 0 ustar kapil kapil 0000000 0000000
from functools import partial
import contextlib
import logging
import time
import zookeeper
from twisted.internet.defer import (
inlineCallbacks, DeferredLock, fail, returnValue, Deferred)
from client import (
ZookeeperClient, ClientEvent, NotConnectedException,
ConnectionTimeoutException)
from retry import RetryClient, is_session_error
from utils import sleep
class StopWatcher(Exception):
pass
BACKOFF_INCREMENT = 10
MAX_BACKOFF = 360
WATCH_KIND_MAP = {
"child": "get_children_and_watch",
"exists": "exists_and_watch",
"get": "get_and_watch"}
log = logging.getLogger("txzk.managed")
#log.setLevel(logging.INFO)
class Watch(object):
"""
For application driven persistent watches, where the application
is manually resetting the watch.
"""
__slots__ = ("_mgr", "_client", "_path", "_kind", "_callback")
def __init__(self, mgr, path, kind, callback):
self._mgr = mgr
self._path = path
self._kind = kind
self._callback = callback
@property
def path(self):
return self._path
@property
def kind(self):
return self._kind
@contextlib.contextmanager
def _ctx(self):
mgr = self._mgr
del self._mgr
try:
yield mgr
finally:
mgr.remove(self)
@inlineCallbacks
def reset(self):
with self._ctx():
yield self._callback(
zookeeper.SESSION_EVENT,
zookeeper.CONNECTED_STATE,
self._path)
def __call__(self, *args, **kw):
with self._ctx():
return self._callback(*args, **kw)
def __str__(self):
return "" % (self.kind, self.path, self._callback)
class WatchManager(object):
watch_class = Watch
def __init__(self):
self._watches = []
def add(self, path, watch_type, watcher):
w = self.watch_class(self, path, watch_type, watcher)
self._watches.append(w)
return w
def remove(self, w):
try:
self._watches.remove(w)
except ValueError:
pass
def iterkeys(self):
for w in self._watches:
yield (w.path, w.kind)
def clear(self):
del self._watches
self._watches = []
@inlineCallbacks
def reset(self, *ignored):
watches = self._watches
self._watches = []
for w in watches:
try:
yield w.reset()
except Exception, e:
log.error("Error reseting watch %s with session event. %s %r",
w, e, e)
continue
class SessionClient(ZookeeperClient):
"""A managed client that automatically re-establishes ephemerals and
triggers watches after reconnecting post session expiration.
This abstracts the client from session expiration handling. It does
come at a cost though.
There are two application constraints that need to be considered for usage
of the SessionClient or ManagedClient. The first is that watch callbacks
which examine the event, must be able to handle the synthetic session
event which is sent to them when the session is re-established.
The second and more problematic is that algorithms/patterns
utilizing ephemeral sequence nodes need to be rethought, as the
session client will recreate the nodes when reconnecting at their
previous paths. Some algorithms (like the std zk lock recipe) rely
on properties like the smallest valued ephemeral sequence node in
a container to identify the lock holder, with the notion that upon
session expiration a new lock/leader will be sought. Sequence
ephemeral node recreation in this context is problematic as the
node is recreated at the exact previous path. Alternative lock
strategies that do work are fairly simple at low volume, such as
owning a particular node path (ie. /locks/lock-holder) with an
ephemeral.
As a result the session client only tracks and restablishes non sequence
ephemeral nodes. For coordination around ephemeral sequence nodes it
provides for watching for the establishment of new sessions via
`subscribe_new_session`
"""
def __init__(self, servers=None, session_timeout=None,
connect_timeout=4000):
"""
"""
super(SessionClient, self).__init__(servers, session_timeout)
self._connect_timeout = connect_timeout
self._watches = WatchManager()
self._ephemerals = {}
self._session_notifications = []
self._reconnect_lock = DeferredLock()
self.set_connection_error_callback(self._cb_connection_error)
self.set_session_callback(self._cb_session_event)
self._backoff_seconds = 0
self._last_reconnect = time.time()
def subscribe_new_session(self):
d = Deferred()
self._session_notifications.append(d)
return d
@inlineCallbacks
def cb_restablish_session(self, e=None, forced=False):
"""Called on intercept of session expiration to create new session.
This will reconnect to zk, re-establish ephemerals, and
trigger watches.
"""
yield self._reconnect_lock.acquire()
log.debug(
"Connection reconnect, lock acquired handle:%d", self.handle)
try:
# If its been explicitly closed, don't re-establish.
if self.handle is None:
log.debug("No handle, client closed")
return
# Don't allow forced reconnect hurds within a session.
if forced and (
(time.time() - self._last_reconnect)
< self.session_timeout / 1000.0):
forced = False
if not forced and not self.unrecoverable:
log.debug("Client already connected, allowing retry")
return
elif self.connected or self.handle >= 0:
self.close()
self.handle = -1
# Re-establish
yield self._cb_restablish_session().addErrback(
self._cb_restablish_errback, e)
except Exception, e:
log.error("error while re-establish %r %s" % (e, e))
finally:
log.debug("Reconnect lock released %s", self)
yield self._reconnect_lock.release()
@inlineCallbacks
def _cb_restablish_session(self):
"""Re-establish a new session, and recreate ephemerals and watches.
"""
# Reconnect
while 1:
log.debug("Reconnect loop - connect timeout %d",
self._connect_timeout)
# If we have some failures, back off
if self._backoff_seconds:
log.debug("Backing off reconnect %d" % self._backoff_seconds)
yield sleep(self._backoff_seconds)
# The client was explicitly closed, abort reconnect.
if self.handle is None:
returnValue(self.handle)
try:
yield self.connect(timeout=self._connect_timeout)
log.info("Restablished connection")
self._last_reconnect = time.time()
except ConnectionTimeoutException:
log.debug("Timeout establishing connection, retrying...")
except zookeeper.ZooKeeperException, e:
log.exception("Error while connecting %r %s" % (e, e))
except Exception, e:
log.info("Reconnect unknown error, aborting: %s", e)
raise
else:
break
if self._backoff_seconds < MAX_BACKOFF:
self._backoff_seconds = min(
self._backoff_seconds + BACKOFF_INCREMENT,
MAX_BACKOFF - 1)
# Recreate ephemerals
items = self._ephemerals.items()
self._ephemerals = {}
for path, e in items:
try:
yield self.create(
path, e['data'], acls=e['acls'], flags=e['flags'])
except zookeeper.NodeExistsException:
log.error("Attempt to create ephemeral node failed %r", path)
# Signal watches
yield self._watches.reset()
# Notify new session observers
notifications = self._session_notifications
self._session_notifications = []
# all good, reset backoff
self._backoff_seconds = 0
for n in notifications:
n.callback(True)
def _cb_restablish_errback(self, err, failure):
"""If there's an error re-establishing the session log it.
"""
log.error("Error while trying to re-establish connection %s\n%s" % (
err, failure))
return failure
@inlineCallbacks
def _cb_connection_error(self, client, error):
"""Convert session expiration to a transient connection error.
Dispatches from api usage error.
"""
if not is_session_error(error):
raise error
log.debug("Connection error detected, delaying retry...")
yield sleep(1)
raise zookeeper.ConnectionLossException
# Dispatch from retry exceed session maximum
def cb_retry_expired(self, error):
log.debug("Persistent retry error, reconnecting...")
return self.cb_restablish_session(forced=True)
# Dispatch from connection events
def _cb_session_event(self, client, event):
if (event.type == zookeeper.SESSION_EVENT and
event.connection_state == zookeeper.EXPIRED_SESSION_STATE):
log.debug("Client session expired event, restablishing")
self.cb_restablish_session()
# Client connected tracker on client operations.
def _check_connected(self, d):
"""Clients are automatically reconnected."""
if self.connected:
return
if self.handle is None:
d.errback(NotConnectedException("Connection closed"))
return d
log.info("Detected dead connection, reconnecting...")
c_d = self.cb_restablish_session()
def after_connected(client):
"""Return a transient connection failure.
The retry client will automatically attempt to retry the operation.
"""
log.debug("Reconnected, returning transient error")
return fail(zookeeper.ConnectionLossException("Retry"))
c_d.addCallback(after_connected)
c_d.chainDeferred(d)
return d
# Dispatch from node watches on session expiration
def _watch_session_wrapper(self, watcher, event_type, conn_state, path):
"""Watch wrapper that diverts session events to a connection callback.
"""
if (event_type == zookeeper.SESSION_EVENT and
conn_state == zookeeper.EXPIRED_SESSION_STATE):
if self.unrecoverable:
log.debug("Watch got session expired event, reconnecting...")
d = self.cb_restablish_session()
d.addErrback(self._cb_restablish_errback)
return d
if event_type == zookeeper.SESSION_EVENT:
if self._session_event_callback:
self._session_event_callback(
self, ClientEvent(
event_type, conn_state, path, self.handle))
else:
return watcher(event_type, conn_state, path)
# Track all watches
def _wrap_watcher(self, watcher, watch_type, path):
if watcher is None:
return watcher
if not callable(watcher):
raise SyntaxError("invalid watcher")
# handle conn watcher, separately.
if watch_type is None and path is None:
return self._zk_thread_callback(
self._watch_session_wrapper, watcher)
return self._zk_thread_callback(
partial(
self._watch_session_wrapper,
self._watches.add(path, watch_type, watcher)))
# Track ephemerals
def _cb_created(self, d, data, acls, flags, result_code, path):
if self._check_result(result_code, d, path=path):
return
if (flags & zookeeper.EPHEMERAL) and not (flags & zookeeper.SEQUENCE):
self._ephemerals[path] = dict(
data=data, acls=acls, flags=flags)
d.callback(path)
def _cb_deleted(self, d, path, result_code):
if self._check_result(result_code, d, path=path):
return
self._ephemerals.pop(path, None)
d.callback(result_code)
def _cb_set_acl(self, d, path, acls, result_code):
if self._check_result(result_code, d, path=path):
return
if path in self._ephemerals:
self._ephemerals[path]['acls'] = acls
d.callback(result_code)
def _cb_set(self, d, path, data, result_code, node_stat):
if self._check_result(result_code, d, path=path):
return
if path in self._ephemerals:
self._ephemerals[path]['data'] = data
d.callback(node_stat)
class _ManagedClient(RetryClient):
def subscribe_new_session(self):
return self.client.subscribe_new_session()
def ManagedClient(servers=None, session_timeout=None, connect_timeout=10000):
client = SessionClient(servers, session_timeout, connect_timeout)
managed_client = _ManagedClient(client)
managed_client.set_retry_error_callback(client.cb_retry_expired)
return managed_client
txzookeeper-0.9.8/txzookeeper/__init__.py 0000664 0001750 0001750 00000001676 12144707107 021040 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from client import ZookeeperClient
__all__ = ["ZookeeperClient"]
# Remember to update debian/changelog as well, for the daily build.
version = "0.9.8"
txzookeeper-0.9.8/txzookeeper/lock.py 0000664 0001750 0001750 00000011333 11745322401 020214 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import zookeeper
from twisted.internet.defer import fail
class LockError(Exception):
"""
A usage or parameter exception that violated the lock conditions.
"""
class Lock(object):
"""
A distributed exclusive lock, based on the apache zookeeper recipe.
http://hadoop.apache.org/zookeeper/docs/r3.3.0/recipes.html
"""
prefix = "lock-"
def __init__(self, path, client):
self._path = path
self._client = client
self._candidate_path = None
self._acquired = False
@property
def path(self):
"""Return the path to the lock."""
return self._path
@property
def acquired(self):
"""Has the lock been acquired. Returns a boolean"""
return self._acquired
def acquire(self):
"""Acquire the lock."""
if self._acquired:
error = LockError("Already holding the lock %s" % (self.path))
return fail(error)
if self._candidate_path is not None:
error = LockError("Already attempting to acquire the lock")
return fail(error)
self._candidate_path = ""
# Create our candidate node in the lock directory.
d = self._client.create(
"/".join((self.path, self.prefix)),
flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
d.addCallback(self._on_candidate_create)
d.addErrback(self._on_no_queue_error)
return d
def _on_candidate_create(self, path):
self._candidate_path = path
return self._acquire()
def _on_no_queue_error(self, failure):
self._candidate_path = None
return failure
def _acquire(self, *args):
d = self._client.get_children(self.path)
d.addCallback(self._check_candidate_nodes)
d.addErrback(self._on_no_queue_error)
return d
def _check_candidate_nodes(self, children):
"""
Check if our lock attempt candidate path is the best candidate
among the list of children names. If it is then we hold the lock
if its not then watch the nearest candidate till it is.
"""
candidate_name = self._candidate_path[
self._candidate_path.rfind('/') + 1:]
# Check to see if our node is the first candidate in the list.
children.sort()
assert candidate_name in children
index = children.index(candidate_name)
if index == 0:
# If our candidate is first, then we already have the lock.
self._acquired = True
return self
# If someone else holds the lock, then wait until holder immediately
# before us releases the lock or dies.
previous_path = "/".join((self.path, children[index - 1]))
exists_deferred, watch_deferred = self._client.exists_and_watch(
previous_path)
exists_deferred.addCallback(
self._check_previous_owner_existence,
watch_deferred)
return exists_deferred
def _check_previous_owner_existence(self, previous_owner_exists,
watch_deferred):
if not previous_owner_exists:
# Hah! It's actually already dead! That was quick. Note
# how we never use the watch deferred in this case.
return self._acquire()
else:
# Nope, there's someone ahead of us in the queue indeed. Let's
# wait for the watch to detect it went away.
watch_deferred.addCallback(self._acquire)
return watch_deferred
def release(self):
"""Release the lock."""
if not self._acquired:
error = LockError("Not holding lock %s" % (self.path))
return fail(error)
d = self._client.delete(self._candidate_path)
def on_delete_success(value):
self._candidate_path = None
self._acquired = False
return True
d.addCallback(on_delete_success)
return d
txzookeeper-0.9.8/txzookeeper/retry.py 0000664 0001750 0001750 00000027454 12144707107 020450 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
"""
A retry client facade that transparently handles transient connection
errors.
"""
import time
import logging
import zookeeper
from twisted.internet.defer import inlineCallbacks, returnValue, Deferred
from txzookeeper.client import NotConnectedException
from txzookeeper.utils import sleep
__all__ = ["retry", "RetryClient"]
log = logging.getLogger("txzk.retry")
class RetryException(Exception):
"""Explicit retry exception.
"""
# Session timeout percentage that we should wait till retrying.
RETRY_FRACTION = 30
def is_retryable(e):
"""Determine if an exception signifies a recoverable connection error.
"""
return isinstance(
e,
(zookeeper.ClosingException,
zookeeper.ConnectionLossException,
zookeeper.OperationTimeoutException,
RetryException))
def is_session_error(e):
return isinstance(
e,
(zookeeper.SessionExpiredException,
zookeeper.ConnectionLossException,
zookeeper.ClosingException,
NotConnectedException))
def _args(args):
return args and args[0] or "NA"
def get_delay(session_timeout, max_delay=5, session_fraction=RETRY_FRACTION):
"""Get retry delay between retrying an operation.
Returns either the specified fraction of a session timeout or the
max delay, whichever is smaller.
The goal is to allow the connection time to auto-heal, before
retrying an operation.
:param session_timeout: The timeout for the session, in milliseconds
:param max_delay: The max delay for a retry, in seconds.
:param session_fraction: The fractional amount of a timeout to wait
"""
retry_delay = (session_timeout * (float(session_fraction) / 100)) / 1000
return min(retry_delay, max_delay)
def check_error(e):
"""Verify a zookeeper connection error, as opposed to an app error.
"""
return is_retryable(e) or is_session_error(e)
def check_retryable(retry_client, max_time, error):
"""Check an error and a client to see if an operation is retryable.
:param retry_client: A txzookeeper client
:param max_time: The max time (epoch tick) that the op is retryable till.
:param error: The client operation exception.
"""
t = time.time()
# Only if the error is known.
if not is_retryable(error):
return False
# Only if we've haven't exceeded the max allotted time.
if max_time <= t:
return False
# Only if the client hasn't been explicitly closed.
if not retry_client.connected:
return False
# Only if the client is in a recoverable state.
if retry_client.unrecoverable:
return False
return True
@inlineCallbacks
def retry(client, func, *args, **kw):
"""Constructs a retry wrapper around a function that retries invocations.
If the function execution results in an exception due to a transient
connection error, the retry wrapper will reinvoke the operation after
a suitable delay (fractional value of the session timeout).
:param client: A ZookeeperClient instance.
:param func: A callable python object that interacts with
zookeeper, the callable must utilize the same zookeeper
connection as passed in the `client` param. The function
must return a single value (either a deferred or result
value).
"""
retry_started = [time.time()]
retry_error = False
while 1:
try:
value = yield func(*args, **kw)
except Exception, e:
# For clients which aren't connected (session timeout == None)
# we raise the errors to the callers.
session_timeout = client.session_timeout or 0
# The longest we keep retrying is 1.2 * session timeout
max_time = (session_timeout / 1000.0) * 1.2 + retry_started[0]
if not check_retryable(client, max_time, e):
# Check if its a persistent client error, and if so use the cb
# if present to try and reconnect for client errors.
if (check_error(e)
and time.time() > max_time
and callable(client.cb_retry_error)
and not retry_error):
log.debug("Retry error %r on %s @ %s",
e, func.__name__, _args(args))
retry_error = True
yield client.cb_retry_error(e)
retry_started[0] = time.time()
continue
raise
# Give the connection a chance to auto-heal.
yield sleep(get_delay(session_timeout))
log.debug("Retry on %s @ %s", func.__name__, _args(args))
continue
returnValue(value)
def retry_watch(client, func, *args, **kw):
"""Contructs a wrapper around a watch callable that retries invocations.
If the callable execution results in an exception due to a transient
connection error, the retry wrapper will reinvoke the operation after
a suitable delay (fractional value of the session timeout).
A watch function must return back a tuple of deferreds
(value_deferred, watch_deferred). No inline callbacks are
performed in here to ensure that callers continue to see a
tuple of results.
The client passed to this retry function must be the same as
the one utilized by the python callable.
:param client: A ZookeeperClient instance.
:param func: A python callable that interacts with zookeeper. If a
function is passed, a txzookeeper client must the first
parameter of this function. The function must return a
tuple of (value_deferred, watch_deferred)
"""
# For clients which aren't connected (session timeout == None)
# we raise the usage errors to the callers
session_timeout = client.session_timeout or 0
# If we keep retrying past the 1.2 * session timeout without
# success just die, the session expiry is fatal.
max_time = session_timeout * 1.2 + time.time()
value_d, watch_d = func(*args, **kw)
def retry_delay(f):
"""Errback, verifes an op is retryable, and delays the next retry.
"""
# Check that operation is retryable.
if not check_retryable(client, max_time, f.value):
return f
# Give the connection a chance to auto-heal
d = sleep(get_delay(session_timeout))
d.addCallback(retry_inner)
return d
def retry_inner(value):
"""Retry operation invoker.
"""
# Invoke the function
retry_value_d, retry_watch_d = func(*args, **kw)
# If we need to retry again.
retry_value_d.addErrback(retry_delay)
# Chain the new watch deferred to the old, presuming its doa
# if the value deferred errored on a connection error.
retry_watch_d.chainDeferred(watch_d)
# Insert back into the callback chain.
return retry_value_d
# Attach the retry
value_d.addErrback(retry_delay)
return value_d, watch_d
def _passproperty(name):
"""Returns a method wrapper that delegates to a client's property.
"""
def wrapper(retry_client):
return getattr(retry_client.client, name)
return property(wrapper)
class RetryClient(object):
"""A ZookeeperClient wrapper that transparently performs retries.
A zookeeper connection can experience transient connection failures
on any operation. As long as the session associated to the connection
is still active on the zookeeper cluster, libzookeeper can reconnect
automatically to the cluster and session and the client is able to
retry.
Whether a given operation is safe for retry depends on the application
in question and how's interacting with zookeeper.
In particular coordination around sequence nodes can be
problematic, as the client has no way of knowing if the operation
succeed or not without additional application specific context.
Idempotent operations against the zookeeper tree are generally
safe to retry.
This class provides a simple wrapper around a zookeeper client,
that will automatically perform retries on operations that
interact with the zookeeper tree, in the face of transient errors,
till the session timeout has been reached. All of the attributes
and methods of a zookeeper client are exposed.
All the methods of the client that interact with the zookeeper tree
are retry enabled.
"""
def __init__(self, client):
self.client = client
self.client.cb_retry_error = None
def set_retry_error_callback(self, callback):
self.client.cb_retry_error = callback
def add_auth(self, *args, **kw):
return retry(self.client, self.client.add_auth, *args, **kw)
def create(self, *args, **kw):
return retry(self.client, self.client.create, *args, **kw)
def delete(self, *args, **kw):
return retry(self.client, self.client.delete, *args, **kw)
def exists(self, *args, **kw):
return retry(self.client, self.client.exists, *args, **kw)
def get(self, *args, **kw):
return retry(self.client, self.client.get, *args, **kw)
def get_acl(self, *args, **kw):
return retry(self.client, self.client.get_acl, *args, **kw)
def get_children(self, *args, **kw):
return retry(self.client, self.client.get_children, *args, **kw)
def set_acl(self, *args, **kw):
return retry(self.client, self.client.set_acl, *args, **kw)
def set(self, *args, **kw):
return retry(self.client, self.client.set, *args, **kw)
def sync(self, *args, **kw):
return retry(self.client, self.client.sync, *args, **kw)
# Watch retries
def exists_and_watch(self, *args, **kw):
return retry_watch(
self.client, self.client.exists_and_watch, *args, **kw)
def get_and_watch(self, *args, **kw):
return retry_watch(
self.client, self.client.get_and_watch, *args, **kw)
def get_children_and_watch(self, *args, **kw):
return retry_watch(
self.client, self.client.get_children_and_watch, *args, **kw)
# Passthrough methods
def set_connection_watcher(self, *args, **kw):
return self.client.set_connection_watcher(*args, **kw)
def set_connection_error_callback(self, *args, **kw):
return self.client.set_connection_error_callback(*args, **kw)
def set_session_callback(self, *args, **kw):
return self.client.set_session_callback(*args, **kw)
def set_determinstic_order(self, *args, **kw):
return self.client.set_determinstic_order(*args, **kw)
def close(self):
return self.client.close()
@inlineCallbacks
def connect(self, *args, **kw):
yield self.client.connect(*args, **kw)
returnValue(self)
# passthrough properties
state = _passproperty("state")
client_id = _passproperty("client_id")
session_timeout = _passproperty("session_timeout")
servers = _passproperty("servers")
handle = _passproperty("handle")
connected = _passproperty("connected")
unrecoverable = _passproperty("unrecoverable")
txzookeeper-0.9.8/txzookeeper/node.py 0000664 0001750 0001750 00000015726 12144707107 020227 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from collections import namedtuple
from zookeeper import NoNodeException, BadVersionException
from twisted.internet.defer import Deferred
from txzookeeper.client import ZOO_OPEN_ACL_UNSAFE
class NodeEvent(namedtuple("NodeEvent", 'type, connection_state, node')):
"""
A node event is returned when a watch deferred fires. It denotes
some event on the zookeeper node that the watch was requested on.
@ivar path: Path to the node the event was about.
@ivar type: An integer corresponding to the event type. The symbolic
name mapping is available from the zookeeper module attributes. For
convience one is included on the C{NodeEvent} class.
@ivar kind: A symbolic name for the event's type.
@ivar connection_state: integer representing the state of the
zookeeper connection.
"""
type_name_map = {
1: 'created',
2: 'deleted',
3: 'changed',
4: 'child'}
@property
def path(self):
return self.node.path
@property
def type_name(self):
return self.type_name_map[self.type]
def __repr__(self):
return "" % (self.type_name, self.path)
class ZNode(object):
"""
A minimal object abstraction over a zookeeper node, utilizes no caching
outside of trying to keep track of node existance and the last read
version. It will attempt to utilize the last read version when modifying
the node. On a bad version exception this values are cleared and the
except reraised (errback chain continue.)
"""
def __init__(self, path, context):
self._path = path
self._name = path.split("/")[-1]
self._context = context
self._node_stat = None
@property
def path(self):
"""
The path to the node from the zookeeper root.
"""
return self._path
@property
def name(self):
"""
The name of the node in its container.
"""
return self._name
def _get_version(self):
if not self._node_stat:
return -1
return self._node_stat["version"]
def _on_error_bad_version(self, failure):
failure.trap(BadVersionException)
self._node_stat = None
return failure
def create(self, data="", acl=None, flags=0):
"""
Create the node with the given data, acl, and persistence flags. If no
acl is given the default public acl is used. The default creation flag
is persistent.
"""
if acl is None:
acl = [ZOO_OPEN_ACL_UNSAFE]
d = self._context.create(self.path, data, acl, flags)
return d
def _on_exists_success(self, node_stat):
if node_stat is None:
return False
self._node_stat = node_stat
return True
def exists(self):
"""
Does the node exist or not, returns a boolean.
"""
d = self._context.exists(self.path)
d.addCallback(self._on_exists_success)
return d
def exists_and_watch(self):
"""
Returns a boolean based on the node's existence. Also returns a
deferred that fires when the node is modified/created/added/deleted.
"""
d, w = self._context.exists_and_watch(self.path)
d.addCallback(self._on_exists_success)
return d, w
def _on_get_node_error(self, failure):
failure.trap(NoNodeException)
self._node_stat = None
return failure
def _on_get_node_success(self, data):
(node_data, node_stat) = data
self._node_stat = node_stat
return node_data
def get_data(self):
"""
Retrieve the node's data.
"""
d = self._context.get(self.path)
d.addCallback(self._on_get_node_success)
d.addErrback(self._on_get_node_error)
return d
def get_data_and_watch(self):
"""
Retrieve the node's data and a deferred that fires when this data
changes.
"""
d, w = self._context.get_and_watch(self.path)
d.addCallback(self._on_get_node_success)
d.addErrback(self._on_get_node_error)
return d, w
def set_data(self, data):
"""Set the node's data."""
def on_success(value):
self._node_stat = None
return self
version = self._get_version()
d = self._context.set(self.path, data, version)
d.addErrback(self._on_error_bad_version)
d.addCallback(on_success)
return d
def get_acl(self):
"""
Get the ACL for this node. An ACL is a list of access control
entity dictionaries.
"""
def on_success((acl, stat)):
if stat is not None:
self._node_stat = stat
return acl
d = self._context.get_acl(self.path)
d.addCallback(on_success)
return d
def set_acl(self, acl):
"""
Set the ACL for this node.
"""
d = self._context.set_acl(
self.path, acl, self._get_version())
d.addErrback(self._on_error_bad_version)
return d
def _on_get_children_filter_results(self, children, prefix):
if prefix:
children = [
name for name in children if name.startswith(prefix)]
return [
self.__class__("/".join((self.path, name)), self._context)
for name in children]
def get_children(self, prefix=None):
"""
Get the children of this node, as ZNode objects. Optionally
a name prefix may be passed which the child node must abide.
"""
d = self._context.get_children(self.path)
d.addCallback(self._on_get_children_filter_results, prefix)
return d
def get_children_and_watch(self, prefix=None):
"""
Get the children of this node, as ZNode objects, and also return
a deferred that fires if a child is added or deleted. Optionally
a name prefix may be passed which the child node must abide.
"""
d, w = self._context.get_children_and_watch(self.path)
d.addCallback(self._on_get_children_filter_results, prefix)
return d, w
def __cmp__(self, other):
return cmp(self.path, other.path)
txzookeeper-0.9.8/txzookeeper/tests/ 0000775 0001750 0001750 00000000000 12144725212 020054 5 ustar kapil kapil 0000000 0000000 txzookeeper-0.9.8/txzookeeper/tests/test_retry.py 0000664 0001750 0001750 00000024623 12144707107 022644 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2012 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import json
import time
import zookeeper
from twisted.internet.defer import inlineCallbacks, fail, succeed, Deferred
from txzookeeper.client import ZookeeperClient
from txzookeeper.retry import (
RetryClient, retry, retry_watch,
check_retryable, is_retryable, get_delay, sleep)
from txzookeeper import retry as retry_module
from txzookeeper.utils import retry_change
from txzookeeper.tests import ZookeeperTestCase, utils
from txzookeeper.tests.proxy import ProxyFactory
from txzookeeper.tests import test_client
class RetryCoreTests(ZookeeperTestCase):
"""Test the retry functions in isolation.
"""
timeout = 5
def test_is_retryable(self):
self.assertEqual(
is_retryable(zookeeper.SessionExpiredException()), False)
self.assertEqual(
is_retryable(zookeeper.ConnectionLossException()), True)
self.assertEqual(
is_retryable(zookeeper.OperationTimeoutException()), True)
self.assertEqual(
is_retryable(TypeError()), False)
def setup_always_retryable(self):
def check_retry(*args):
return True
self.patch(retry_module, "check_retryable", check_retry)
@inlineCallbacks
def test_sleep(self):
t = time.time()
yield sleep(0.5)
self.assertTrue(time.time() - t > 0.5)
@inlineCallbacks
def test_retry_function(self):
"""The retry wrapper can be used for a function."""
self.setup_always_retryable()
results = [fail(zookeeper.ConnectionLossException()),
fail(zookeeper.ConnectionLossException()),
succeed(21)]
def original(zebra):
"""Hello World"""
return results.pop(0)
result = yield retry(ZookeeperClient(), original, "magic")
self.assertEqual(result, 21)
@inlineCallbacks
def test_retry_method(self):
"""The retry wrapper can be used for a method."""
self.setup_always_retryable()
results = [fail(zookeeper.ConnectionLossException()),
fail(zookeeper.ConnectionLossException()),
succeed(21)]
class Foobar(object):
def original(self, zebra):
"""Hello World"""
return results.pop(0)
client = ZookeeperClient()
foo = Foobar()
result = yield retry(client, foo.original, "magic")
self.assertEqual(result, 21)
@inlineCallbacks
def test_retry_watch_function(self):
self.setup_always_retryable()
results = [(fail(zookeeper.ConnectionLossException()), Deferred()),
(fail(zookeeper.ConnectionLossException()), Deferred()),
(succeed(21), succeed(22))]
def original(zebra):
"""Hello World"""
return results.pop(0)
client = ZookeeperClient()
value_d, watch_d = retry_watch(client, original, "magic")
self.assertEqual((yield value_d), 21)
self.assertEqual((yield watch_d), 22)
def test_check_retryable(self):
unrecoverable_errors = [
zookeeper.ApiErrorException(),
zookeeper.NoAuthException(),
zookeeper.NodeExistsException(),
zookeeper.SessionExpiredException(),
]
class _Conn(object):
def __init__(self, **kw):
self.__dict__.update(kw)
error = zookeeper.ConnectionLossException()
conn = _Conn(connected=True, unrecoverable=False)
max_time = time.time() + 10
for e in unrecoverable_errors:
self.assertFalse(check_retryable(_Conn(), max_time, e))
self.assertTrue(check_retryable(conn, max_time, error))
self.assertFalse(check_retryable(conn, time.time() - 10, error))
self.assertFalse(check_retryable(
_Conn(connected=False, unrecoverable=False), max_time, error))
self.assertFalse(check_retryable(
_Conn(connected=True, unrecoverable=True), max_time, error))
def test_get_delay(self):
# Delay currently set to ~1/3 of session time.
# Verify max value is respected
self.assertEqual(get_delay(25000, 10), 7.5)
# Verify normal calculation
self.assertEqual(get_delay(15000, 10, 30), 4.5)
class RetryClientTests(test_client.ClientTests):
"""Run the full client test suite against the retry facade.
"""
def setUp(self):
super(RetryClientTests, self).setUp()
self.client = RetryClient(ZookeeperClient("127.0.0.1:2181", 3000))
self.client2 = None
def tearDown(self):
if self.client.connected:
utils.deleteTree(handle=self.client.handle)
self.client.close()
if self.client2 and self.client2.connected:
self.client2.close()
super(RetryClientTests, self).tearDown()
def test_wb_connect_after_timeout(self):
"""white box tests disabled for retryclient."""
def test_wb_reconnect_after_timeout_and_close(self):
"""white box tests disabled for retryclient."""
def test_exists_with_error(self):
"""White box tests disabled for retryclient."""
class RetryClientConnectionLossTest(ZookeeperTestCase):
def setUp(self):
super(RetryClientConnectionLossTest, self).setUp()
from twisted.internet import reactor
self.proxy = ProxyFactory("127.0.0.1", 2181)
self.proxy_port = reactor.listenTCP(0, self.proxy)
host = self.proxy_port.getHost()
self.proxied_client = RetryClient(ZookeeperClient(
"%s:%s" % (host.host, host.port)))
self.direct_client = ZookeeperClient("127.0.0.1:2181", 3000)
self.session_events = []
def session_event_collector(conn, event):
self.session_events.append(event)
self.proxied_client.set_session_callback(session_event_collector)
return self.direct_client.connect()
@inlineCallbacks
def tearDown(self):
import zookeeper
zookeeper.set_debug_level(0)
if self.proxied_client.connected:
yield self.proxied_client.close()
if not self.direct_client.connected:
yield self.direct_client.connect()
utils.deleteTree(handle=self.direct_client.handle)
yield self.direct_client.close()
self.proxy.lose_connection()
yield self.proxy_port.stopListening()
@inlineCallbacks
def test_get_children_and_watch(self):
yield self.proxied_client.connect()
# Setup tree
cpath = "/test-tree"
yield self.direct_client.create(cpath)
# Block the request (drops all packets.)
self.proxy.set_blocked(True)
child_d, watch_d = self.proxied_client.get_children_and_watch(cpath)
# Unblock and disconnect
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# Call goes through
self.assertEqual((yield child_d), [])
self.assertEqual(len(self.session_events), 2)
# And we have reconnect events
self.assertEqual(self.session_events[-1].state_name, "connected")
yield self.direct_client.create(cpath + "/abc")
# The original watch is still active
yield watch_d
@inlineCallbacks
def test_exists_and_watch(self):
yield self.proxied_client.connect()
cpath = "/test-tree"
# Block the request
self.proxy.set_blocked(True)
exists_d, watch_d = self.proxied_client.exists_and_watch(cpath)
# Create the node
yield self.direct_client.create(cpath)
# Unblock and disconnect
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# Call gets retried, see the latest state
self.assertTrue((yield exists_d))
self.assertEqual(len(self.session_events), 2)
# And we have reconnect events
self.assertEqual(self.session_events[-1].state_name, "connected")
yield self.direct_client.delete(cpath)
# The original watch is still active
yield watch_d
@inlineCallbacks
def test_get_and_watch(self):
yield self.proxied_client.connect()
# Setup tree
cpath = "/test-tree"
yield self.direct_client.create(cpath)
# Block the request (drops all packets.)
self.proxy.set_blocked(True)
get_d, watch_d = self.proxied_client.get_and_watch(cpath)
# Unblock and disconnect
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# Call goes through
content, stat = yield get_d
self.assertEqual(content, '')
self.assertEqual(len(self.session_events), 2)
# And we have reconnect events
self.assertEqual(self.session_events[-1].state_name, "connected")
yield self.direct_client.delete(cpath)
# The original watch is still active
yield watch_d
@inlineCallbacks
def test_set(self):
yield self.proxied_client.connect()
# Setup tree
cpath = "/test-tree"
yield self.direct_client.create(cpath, json.dumps({"a": 1, "c": 2}))
def update_node(content, stat):
data = json.loads(content)
data["a"] += 1
data["b"] = 0
return json.dumps(data)
# Block the request (drops all packets.)
self.proxy.set_blocked(True)
mod_d = retry_change(self.proxied_client, cpath, update_node)
# Unblock and disconnect
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# Call goes through, contents verified.
yield mod_d
content, stat = yield self.direct_client.get(cpath)
self.assertEqual(json.loads(content),
{"a": 2, "b": 0, "c": 2})
txzookeeper-0.9.8/txzookeeper/tests/test_managed.py 0000664 0001750 0001750 00000023761 12144707107 023075 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2012 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import logging
import zookeeper
from twisted.internet.defer import inlineCallbacks, Deferred, DeferredList
from txzookeeper.client import ZookeeperClient, ClientEvent
from txzookeeper import managed
from txzookeeper.tests import ZookeeperTestCase, utils
from txzookeeper.tests import test_client
class WatchTest(ZookeeperTestCase):
"""Watch manager and watch tests"""
def setUp(self):
self.watches = managed.WatchManager()
def tearDown(self):
self.watches.clear()
del self.watches
def test_add_remove(self):
w = self.watches.add("/foobar", "child", lambda x: 1)
self.assertIn(
"")
d.callback(True)
yield reset_done
self.assertNotIn(w, self.watches._watches)
self.assertIn("Error reseting watch", output.getvalue())
@inlineCallbacks
def test_reset(self):
"""Reset fires a synthentic client event, and clears watches.
"""
d = Deferred()
results = []
def callback(*args, **kw):
results.append((args, kw))
return d
w = self.watches.add("/foobar", "child", callback)
reset_done = self.watches.reset()
e, _ = results.pop()
e = list(e)
e.append(0)
self.assertEqual(
str(ClientEvent(*e)),
"")
d.callback(True)
yield reset_done
self.assertNotIn(w, self.watches._watches)
class SessionClientTests(test_client.ClientTests):
"""Run through basic operations with SessionClient."""
timeout = 5
def setUp(self):
super(SessionClientTests, self).setUp()
self.client = managed.SessionClient("127.0.0.1:2181")
def test_client_use_while_disconnected_returns_failure(self):
# managed session client reconnects here.
return True
class SessionClientExpireTests(ZookeeperTestCase):
"""Verify expiration behavior."""
def setUp(self):
super(SessionClientExpireTests, self).setUp()
self.client = managed.ManagedClient("127.0.0.1:2181", 3000)
self.client2 = None
self.output = self.capture_log(level=logging.DEBUG)
return self.client.connect()
@inlineCallbacks
def tearDown(self):
self.client.close()
self.client2 = ZookeeperClient("127.0.0.1:2181")
yield self.client2.connect()
utils.deleteTree(handle=self.client2.handle)
yield self.client2.close()
super(SessionClientExpireTests, self).tearDown()
@inlineCallbacks
def expire_session(self, wait=True, retry=0):
assert self.client.connected
#if wait:
# d = self.client.subscribe_new_session()
client_id = self.client.client_id
self.client2 = ZookeeperClient(self.client.servers)
yield self.client2.connect(client_id=client_id)
yield self.client2.close()
# It takes some time to propagate (1/3 session time as ping)
if wait:
yield self.sleep(2)
client_new_id = self.client.client_id
# Crappy workaround to c libzk bug/issue see http://goo.gl/9ei5c
# Works most of the time.. but bound it when it doesn't. lame!
if client_id[0] == client_new_id[0] and retry < 10:
yield self.expire_session(wait, retry+1)
@inlineCallbacks
def test_session_expiration_conn(self):
d = self.client.subscribe_new_session()
session_id = self.client.client_id[0]
yield self.client.create("/fo-1", "abc")
yield self.expire_session(wait=True)
yield d
stat = yield self.client.exists("/")
self.assertTrue(stat)
self.assertNotEqual(session_id, self.client.client_id[0])
@inlineCallbacks
def test_session_expiration_notification(self):
session_id = self.client.client_id[0]
c_d, w_d = self.client.get_and_watch("/")
yield c_d
d = self.client.subscribe_new_session()
self.assertFalse(d.called)
yield self.expire_session(wait=True)
yield d
yield w_d
self.assertNotEqual(session_id, self.client.client_id[0])
@inlineCallbacks
def test_invoked_watch_gc(self):
c_d, w_d = yield self.client.get_children_and_watch("/")
yield c_d
yield self.client.create("/foo")
yield w_d
yield self.expire_session()
yield self.client.create("/foo2")
# Nothing should blow up
yield self.sleep(0.2)
@inlineCallbacks
def test_app_usage_error_bypass_retry(self):
"""App errors shouldn't trigger a reconnect."""
output = self.capture_log(level=logging.DEBUG)
yield self.assertFailure(
self.client.get("/abc"), zookeeper.NoNodeException)
self.assertNotIn("Persistent retry error", output.getvalue())
@inlineCallbacks
def test_ephemeral_and_watch_recreate(self):
# Create some ephemeral nodes
yield self.client.create("/fo-1", "abc", flags=zookeeper.EPHEMERAL)
yield self.client.create("/fo-2", "def", flags=zookeeper.EPHEMERAL)
# Create some watches
g_d, g_w_d = self.client.get_and_watch("/fo-1")
yield g_d
c_d, c_w_d = self.client.get_children_and_watch("/")
yield g_d
e_d, e_w_d = self.client.get_children_and_watch("/fo-2")
yield e_d
# Expire the session
yield self.expire_session()
# Poof
# Ephemerals back
c, s = yield self.client.get("/fo-1")
self.assertEqual(c, "abc")
c, s = yield self.client.get("/fo-2")
self.assertEqual(c, "def")
# Watches triggered
yield DeferredList(
[g_w_d, c_w_d, e_w_d],
fireOnOneErrback=True, consumeErrors=True)
h = self.client.handle
self.assertEqual(
[str(d.result) for d in (g_w_d, c_w_d, e_w_d)],
["" % h,
"" % h,
"" % h
])
@inlineCallbacks
def test_ephemeral_no_track_sequence_nodes(self):
""" Ephemeral tracking ignores sequence nodes.
"""
yield self.client.create("/music", "abc")
yield self.client.create(
"/music/u2-", "abc",
flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
yield self.expire_session()
children = yield self.client.get_children("/music")
self.assertEqual(children, [])
@inlineCallbacks
def test_ephemeral_content_modification(self):
yield self.client.create("/fo-1", "abc", flags=zookeeper.EPHEMERAL)
yield self.client.set("/fo-1", "def")
yield self.expire_session()
c, s = yield self.client.get("/fo-1")
self.assertEqual(c, "def")
@inlineCallbacks
def test_ephemeral_acl_modification(self):
yield self.client.create("/fo-1", "abc", flags=zookeeper.EPHEMERAL)
acl = [test_client.PUBLIC_ACL,
dict(scheme="digest",
id="zebra:moon",
perms=zookeeper.PERM_ALL)]
yield self.client.set_acl("/fo-1", acl)
yield self.expire_session()
n_acl, stat = yield self.client.get_acl("/fo-1")
self.assertEqual(acl, n_acl)
@inlineCallbacks
def test_ephemeral_deletion(self):
yield self.client.create("/fo-1", "abc", flags=zookeeper.EPHEMERAL)
yield self.client.delete("/fo-1")
yield self.expire_session()
self.assertFalse((yield self.client.exists("/fo-1")))
txzookeeper-0.9.8/txzookeeper/tests/__init__.py 0000664 0001750 0001750 00000006212 12144707107 022171 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import logging
import StringIO
import sys
import zookeeper
from twisted.internet.defer import Deferred
from twisted.trial.unittest import TestCase
from mocker import MockerTestCase
class ZookeeperTestCase(TestCase, MockerTestCase):
def setUp(self):
super(ZookeeperTestCase, self).setUp()
zookeeper.set_debug_level(0)
def tearDown(self):
super(ZookeeperTestCase, self).tearDown()
def get_log(self):
return open(self.log_file_path).read()
def sleep(self, delay):
"""Non-blocking sleep."""
from twisted.internet import reactor
deferred = Deferred()
reactor.callLater(delay, deferred.callback, None)
return deferred
_missing_attr = object()
def patch(self, object, attr, value):
"""Replace an object's attribute, and restore original value later.
Returns the original value of the attribute if any or None.
"""
original_value = getattr(object, attr, self._missing_attr)
@self.addCleanup
def restore_original():
if original_value is self._missing_attr:
try:
delattr(object, attr)
except AttributeError:
pass
else:
setattr(object, attr, original_value)
setattr(object, attr, value)
if original_value is self._missing_attr:
return None
return original_value
def capture_log(self, name="", level=logging.INFO,
log_file=None, formatter=None):
"""Capture log channel to StringIO"""
if log_file is None:
log_file = StringIO.StringIO()
log_handler = logging.StreamHandler(log_file)
if formatter:
log_handler.setFormatter(formatter)
logger = logging.getLogger(name)
logger.addHandler(log_handler)
old_logger_level = logger.level
logger.setLevel(level)
@self.addCleanup
def reset_logging():
logger.removeHandler(log_handler)
logger.setLevel(old_logger_level)
return log_file
def egg_test_runner():
"""
Test collector and runner for setup.py test
"""
from twisted.scripts.trial import run
original_args = list(sys.argv)
sys.argv = ["", "txzookeeper"]
try:
return run()
finally:
sys.argv = original_args
txzookeeper-0.9.8/txzookeeper/tests/test_lock.py 0000664 0001750 0001750 00000017341 11745322401 022422 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from twisted.internet.defer import (
inlineCallbacks, returnValue, Deferred, succeed)
from zookeeper import NoNodeException
from txzookeeper import ZookeeperClient
from txzookeeper.lock import Lock, LockError
from mocker import ANY
from txzookeeper.tests import ZookeeperTestCase, utils
class LockTests(ZookeeperTestCase):
def setUp(self):
super(LockTests, self).setUp()
self.clients = []
def tearDown(self):
cleanup = False
for client in self.clients:
if not cleanup and client.connected:
utils.deleteTree(handle=client.handle)
cleanup = True
client.close()
@inlineCallbacks
def open_client(self, credentials=None):
"""
Open a zookeeper client, optionally authenticating with the
credentials if given.
"""
client = ZookeeperClient("127.0.0.1:2181")
self.clients.append(client)
yield client.connect()
if credentials:
d = client.add_auth("digest", credentials)
# hack to keep auth fast
yield client.exists("/")
yield d
returnValue(client)
@inlineCallbacks
def test_acquire_release(self):
"""
A lock can be acquired and released.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
released = yield lock.release()
self.assertEqual(released, True)
@inlineCallbacks
def test_lock_reuse(self):
"""
A lock instance may be reused after an acquire/release cycle.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
yield lock.acquire()
self.assertTrue(lock.acquired)
yield lock.release()
self.assertFalse(lock.acquired)
@inlineCallbacks
def test_error_on_double_acquire(self):
"""
Attempting to acquire an already held lock, raises a Value Error.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
yield self.failUnlessFailure(lock.acquire(), LockError)
@inlineCallbacks
def test_acquire_after_error(self):
"""
Any instance state associated with a failed acquired should be cleared
on error, allowing subsequent to succeed.
"""
client = yield self.open_client()
path = "/lock-test-acquire-after-error"
lock = Lock(path, client)
d = lock.acquire()
self.failUnlessFailure(d, NoNodeException)
yield d
yield client.create(path)
yield lock.acquire()
self.assertEqual(lock.acquired, True)
@inlineCallbacks
def test_error_on_acquire_acquiring(self):
"""
Attempting to acquire the lock while an attempt is already in progress,
raises a LockError.
"""
client = yield self.open_client()
path = yield client.create("/lock-test")
lock = Lock(path, client)
# setup the client to create the intended environment
mock_client = self.mocker.patch(client)
mock_client.create(ANY, flags=ANY)
self.mocker.result(succeed("%s/%s" % (path, "lock-3")))
mock_client.get_children("/lock-test")
self.mocker.result(succeed(["lock-2", "lock-3"]))
mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
watch = Deferred()
self.mocker.result((succeed(True), watch))
self.mocker.replay()
# now we attempt to acquire the lock, rigged above to not succeed
d = lock.acquire()
test_deferred = Deferred()
# and next we schedule a lock attempt, which should fail as we're
# still attempting to acquire the lock.
def attempt_acquire():
# make sure lock was previously attempted acquired without
# error (disregarding that it was rigged to *fail*)
from twisted.python.failure import Failure
self.assertFalse(isinstance(d.result, Failure))
# acquire lock and expect to fail
self.failUnlessFailure(lock.acquire(), LockError)
# after we've verified the error handling, end the test
test_deferred.callback(None)
from twisted.internet import reactor
reactor.callLater(0.1, attempt_acquire)
yield test_deferred
@inlineCallbacks
def test_no_previous_owner_bypasses_watch(self):
"""
Coverage test. Internally the lock algorithm checks and sets a
watch on the nearest candidate node. If the node has been removed
between the time between the get_children and exists call, the we
immediately reattempt to get the lock without waiting on the watch.
"""
client = yield self.open_client()
path = yield client.create("/lock-no-previous")
# setup the client to create the intended environment
mock_client = self.mocker.patch(client)
mock_client.create(ANY, flags=ANY)
self.mocker.result(succeed("%s/%s" % (path, "lock-3")))
mock_client.get_children(path)
self.mocker.result(succeed(["lock-2", "lock-3"]))
mock_client.exists_and_watch("%s/%s" % (path, "lock-2"))
watch = Deferred()
self.mocker.result((succeed(False), watch))
mock_client.get_children(path)
self.mocker.result(succeed(["lock-3"]))
self.mocker.replay()
lock = Lock(path, mock_client)
yield lock.acquire()
self.assertTrue(lock.acquired)
@inlineCallbacks
def test_error_when_releasing_unacquired(self):
"""
If an attempt is made to release a lock, that not currently being held,
than a C{LockError} exception is raised.
"""
client = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
self.failUnlessFailure(lock.release(), LockError)
@inlineCallbacks
def test_multiple_acquiring_clients(self):
"""
Multiple clients can compete for the lock, only one client's Lock
instance may hold the lock at any given moment.
"""
client = yield self.open_client()
client2 = yield self.open_client()
lock_dir = yield client.create("/lock-multi-test")
lock = Lock(lock_dir, client)
lock2 = Lock(lock_dir, client2)
yield lock.acquire()
self.assertTrue(lock.acquired)
lock2_acquire = lock2.acquire()
yield lock.release()
yield lock2_acquire
self.assertTrue(lock2.acquired)
self.assertFalse(lock.acquired)
txzookeeper-0.9.8/txzookeeper/tests/test_security.py 0000664 0001750 0001750 00000024121 12106345553 023340 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import hashlib
import base64
import zookeeper
from twisted.internet.defer import inlineCallbacks, returnValue
from twisted.python.failure import Failure
from txzookeeper.tests import ZookeeperTestCase, utils
from txzookeeper.client import (ZookeeperClient, ZOO_OPEN_ACL_UNSAFE)
PUBLIC_ACL = ZOO_OPEN_ACL_UNSAFE
class SecurityTests(ZookeeperTestCase):
ident_bob = "bob:bob"
ident_alice = "alice:alice"
ident_eve = "eve:eve"
ident_chuck = "chuck:chuck"
ident_unittest = "unittest:unittest"
def setUp(self):
super(SecurityTests, self).setUp()
self.clients = []
self.test_cleanup_connection = ZookeeperClient("127.0.0.1:2181", 2000)
self.access_control_test_cleanup_entry = self.make_ac(
self.ident_unittest, all=True, admin=True)
return self.open_and_authenticate(
self.test_cleanup_connection, self.ident_unittest)
def tearDown(self):
utils.deleteTree(handle=self.test_cleanup_connection.handle)
for client in self.clients:
if client.connected:
client.close()
self.test_cleanup_connection.close()
@inlineCallbacks
def open_and_authenticate(self, client, credentials):
"""authentication so the test always has access to clean up the
zookeeper node tree. synchronous auth to avoid using deferred
during setup."""
yield client.connect()
d = client.add_auth("digest", credentials)
# hack to keep auth fast
yield client.exists("/")
yield d
returnValue(client)
@inlineCallbacks
def connect_users(self, *users):
clients = []
for name in users:
ident_user = getattr(self, "ident_%s" % (name), None)
if ident_user is None:
raise AttributeError("Invalid User %s" % (name))
client = ZookeeperClient("127.0.0.1:2181", 3000)
clients.append(client)
yield self.open_and_authenticate(client, ident_user)
self.clients.extend(clients)
returnValue(clients)
@inlineCallbacks
def sync_clients(self, *clients):
for client in clients:
yield client.sync()
def ensure_auth_failure(self, result):
if isinstance(result, Failure):
self.assertTrue(isinstance(
result.value, zookeeper.NoAuthException))
return
self.fail("should have raised auth exception")
def make_acl(self, *access_control_entries):
"""
Take the variable number of access control entries and return a
list suitable for passing to the txzookeeper's api as an ACL.
Also automatically appends the test acess control entry to ensure
that the test can cleanup regardless of node permissions set within
a test.
"""
access_control_list = list(access_control_entries)
access_control_list.append(self.access_control_test_cleanup_entry)
return access_control_list
def make_ac(self, credentials, **kw):
"""
Given a username:password credential and boolean keyword arguments
corresponding to permissions construct an access control entry.
"""
user, password = credentials.split(":")
identity = "%s:%s" % (
user,
base64.b64encode(hashlib.new('sha1', credentials).digest()))
permissions = None
for name, perm in (('read', zookeeper.PERM_READ),
('write', zookeeper.PERM_WRITE),
('delete', zookeeper.PERM_DELETE),
('create', zookeeper.PERM_CREATE),
('admin', zookeeper.PERM_ADMIN),
('all', zookeeper.PERM_ALL)):
if name not in kw:
continue
if permissions is None:
permissions = perm
else:
permissions = permissions | perm
if permissions is None:
raise SyntaxError("No permissions specified")
access_control_entry = {
'id': identity, 'scheme': 'digest', 'perms': permissions}
return access_control_entry
@inlineCallbacks
def test_bob_message_for_alice_with_eve_reading(self):
"""
If bob creates a message for alice to read, eve cannot read
it.
"""
bob, alice, eve = yield self.connect_users(
"bob", "alice", "eve")
yield bob.create(
"/message_inbox", "message for alice",
self.make_acl(
self.make_ac(self.ident_bob, write=True, read=True),
self.make_ac(self.ident_alice, read=True)))
message_content, message_stat = yield alice.get("/message_inbox")
self.assertEqual(message_content, "message for alice")
d = eve.get("/message_inbox")
d.addBoth(self.ensure_auth_failure)
yield d
@inlineCallbacks
def test_alice_message_box_for_bob_with_eve_deleting(self):
"""
If alice makes a folder to drop off messages to bob, neither bob nor
eve can write to it, and bob can only read, and delete the messages.
The permission for deleting is set on the container node. Bob has
delete permission only on the on the container, and can delete nodes.
Even if eve has permission to delete on the message node, without the
container permission it will not succeed.
"""
bob, alice, eve = yield self.connect_users("bob", "alice", "eve")
yield alice.create(
"/from_alice", "messages from alice",
self.make_acl(
self.make_ac(self.ident_alice, create=True, write=True),
self.make_ac(self.ident_bob, read=True, delete=True))),
# make sure all the clients have a consistent view
yield self.sync_clients(alice, bob, eve)
# bob can't create messages in the mailbox
d = bob.create("/from_alice/love_letter", "test")
d.addBoth(self.ensure_auth_failure)
# alice's message can only be read by bob.
path = yield alice.create(
"/from_alice/appreciate_letter", "great",
self.make_acl(
self.make_ac(self.ident_eve, delete=True),
self.make_ac(self.ident_bob, read=True),
self.make_ac(self.ident_alice, create=True, write=True)))
message_content, node_stat = yield bob.get(path)
self.assertEqual(message_content, "great")
# make sure all the clients have a consistent view
yield self.sync_clients(alice, bob, eve)
# eve can neither read nor delete
d = eve.get(path)
d.addBoth(self.ensure_auth_failure)
yield d
d = eve.delete(path)
d.addBoth(self.ensure_auth_failure)
yield d
# bob can delete the message when he's done reading.
yield bob.delete(path)
def test_eve_can_discover_node_path(self):
"""
One weakness of the zookeeper security model, is that it enables
discovery of a node existance, its node stats, and its acl to
any inquiring party.
The acl is read off the node and then used as enforcement to any
policy. Ideally it should validate exists and get_acl against
the read permission on the node.
Here bob creates a node that only he can read or write to, but
eve can still get node stat on the node if she knows the path.
"""
bob, eve = yield self.connect_users("bob", "eve")
yield bob.create("/bobsafeplace", "",
self.make_acl(self.make_ac(self.ident_bob, all=True)))
yield bob.create("/bobsafeplace/secret-a", "supersecret",
self.make_acl(self.make_ac(self.ident_bob, all=True)))
self.sync_clients(bob, eve)
d = eve.exists("/bobsafeplace")
def verify_node_stat(node_stat):
self.assertEqual(node_stat["dataLength"], len("supersecret"))
self.assertEqual(node_stat["version"], 0)
d.addCallback(verify_node_stat)
yield d
def test_eve_can_discover_node_acl(self):
"""
One weakness of the zookeeper security model, is that it enables
discovery of a node existance, its node stats, and its acl to
any inquiring party.
The acl is read off the node and then used as enforcement to any
policy. Ideally it should validate exists and get_acl against
the read permission on the node.
Here bob creates a node that only he can read or write to, but
eve can still get node stat and acl information on the node if
she knows the path.
"""
bob, eve = yield self.connect_users("bob", "eve")
yield bob.create("/bobsafeplace", "",
self.make_acl(self.make_ac(self.ident_bob, all=True)))
yield bob.create("/bobsafeplace/secret-a", "supersecret",
self.make_acl(self.make_ac(self.ident_bob, all=True)))
self.sync_clients(bob, eve)
d = eve.get_acl("/bobsafeplace/secret-a")
def verify_node_stat_and_acl((acl, node_stat)):
self.assertEqual(node_stat["dataLength"], len("supersecret"))
self.assertEqual(node_stat["version"], 0)
self.assertEqual(acl[0]["id"].split(":")[0], "bob")
d.addCallback(verify_node_stat_and_acl)
yield d
txzookeeper-0.9.8/txzookeeper/tests/test_session.py 0000664 0001750 0001750 00000027771 12144707107 023171 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import atexit
import logging
import os
import zookeeper
from twisted.internet.defer import (
inlineCallbacks, Deferred, DeferredList, returnValue)
from txzookeeper import ZookeeperClient
from txzookeeper.client import NotConnectedException, ConnectionException
from txzookeeper.tests.common import ZookeeperCluster
from txzookeeper.tests import ZookeeperTestCase
from txzookeeper import managed
ZK_HOME = os.environ.get("ZOOKEEPER_PATH")
assert ZK_HOME, (
"ZOOKEEPER_PATH environment variable must be defined.\n "
"For deb package installations this is /usr/share/java")
CLUSTER = ZookeeperCluster(ZK_HOME)
atexit.register(lambda cluster: cluster.terminate(), CLUSTER)
class ClientSessionTests(ZookeeperTestCase):
def setUp(self):
super(ClientSessionTests, self).setUp()
self.cluster.start()
self.client = None
self.client2 = None
zookeeper.deterministic_conn_order(True)
zookeeper.set_debug_level(0)
@property
def cluster(self):
return CLUSTER
def tearDown(self):
super(ClientSessionTests, self).tearDown()
if self.client:
self.client.close()
if self.client2:
self.client2.close()
self.cluster.reset()
@inlineCallbacks
def test_client_session_migration(self):
"""A client will automatically rotate servers to ensure a connection.
A client connected to multiple servers, will transparently
migrate amongst them, as individual servers can no longer be
reached. A client's session will be maintined.
"""
# Connect to the Zookeeper Cluster
servers = ",".join([s.address for s in self.cluster])
self.client = ZookeeperClient(servers)
yield self.client.connect()
yield self.client.create("/hello", flags=zookeeper.EPHEMERAL)
# Shutdown the server the client is connected to
self.cluster[0].stop()
# Wait for the shutdown and cycle, if we don't wait we'll
# get a zookeeper connectionloss exception on occassion.
yield self.sleep(0.1)
self.assertTrue(self.client.connected)
exists = yield self.client.exists("/hello")
self.assertTrue(exists)
@inlineCallbacks
def test_client_watch_migration(self):
"""On server rotation, extant watches are still active.
A client connected to multiple servers, will transparently
migrate amongst them, as individual servers can no longer be
reached. Watch deferreds issued from the same client instance will
continue to function as the session is maintained.
"""
session_events = []
def session_event_callback(connection, e):
session_events.append(e)
# Connect to the Zookeeper Cluster
servers = ",".join([s.address for s in self.cluster])
self.client = ZookeeperClient(servers)
self.client.set_session_callback(session_event_callback)
yield self.client.connect()
# Setup a watch
yield self.client.create("/hello")
exists_d, watch_d = self.client.exists_and_watch("/hello")
yield exists_d
# Shutdown the server the client is connected to
self.cluster[0].stop()
# Wait for the shutdown and cycle, if we don't wait we'll
# get occasionally get a zookeeper connectionloss exception.
yield self.sleep(0.1)
# The session events that would have been ignored are sent
# to the session event callback.
self.assertTrue(session_events)
self.assertTrue(self.client.connected)
# If we delete the node, we'll see the watch fire.
yield self.client.delete("/hello")
event = yield watch_d
self.assertEqual(event.type_name, "deleted")
self.assertEqual(event.path, "/hello")
@inlineCallbacks
def test_connection_error_handler(self):
"""A callback can be specified for connection errors.
We can specify a callback for connection errors, that
can perform recovery for a disconnected client.
"""
@inlineCallbacks
def connection_error_handler(connection, error):
# Moved management of this connection attribute out of the
# default behavior for a connection exception, to support
# the retry facade. Under the hood libzk is going to be
# trying to transparently reconnect
connection.connected = False
# On loss of the connection, reconnect the client w/ same session.
connection.close()
yield connection.connect(
self.cluster[1].address, client_id=connection.client_id)
returnValue(23)
self.client = ZookeeperClient(self.cluster[0].address)
self.client.set_connection_error_callback(connection_error_handler)
yield self.client.connect()
yield self.client.create("/hello")
exists_d, watch_d = self.client.exists_and_watch("/hello")
yield exists_d
# Shutdown the server the client is connected to
self.cluster[0].stop()
yield self.sleep(0.1)
# Results in connection loss exception, and invoking of error handler.
result = yield self.client.exists("/hello")
# The result of the error handler is returned to the api
self.assertEqual(result, 23)
exists = yield self.client.exists("/hello")
self.assertTrue(exists)
@inlineCallbacks
def test_client_session_expiration_event(self):
"""A client which recieves a session expiration event.
"""
session_events = []
events_received = Deferred()
def session_event_callback(connection, e):
session_events.append(e)
if len(session_events) == 8:
events_received.callback(True)
# Connect to a node in the cluster and establish a watch
self.client = ZookeeperClient(self.cluster[0].address)
self.client.set_session_callback(session_event_callback)
yield self.client.connect()
# Setup some watches to verify they are cleaned out on expiration.
d, e_watch_d = self.client.exists_and_watch("/")
yield d
d, g_watch_d = self.client.get_and_watch("/")
yield d
d, c_watch_d = self.client.get_children_and_watch("/")
yield d
# Connect a client to the same session on a different node.
self.client2 = ZookeeperClient(self.cluster[1].address)
yield self.client2.connect(client_id=self.client.client_id)
# Close the new client and wait for the event propogation
yield self.client2.close()
# It can take some time for this to propagate
yield events_received
self.assertEqual(len(session_events), 8)
# The last four (conn + 3 watches) are all expired
for evt in session_events[4:]:
self.assertEqual(evt.state_name, "expired")
# The connection is dead without reconnecting.
yield self.assertFailure(
self.client.exists("/"),
NotConnectedException, ConnectionException)
self.assertTrue(self.client.unrecoverable)
yield self.assertFailure(e_watch_d, zookeeper.SessionExpiredException)
yield self.assertFailure(g_watch_d, zookeeper.SessionExpiredException)
yield self.assertFailure(c_watch_d, zookeeper.SessionExpiredException)
# If a reconnect attempt is made with a dead session id
client_id = self.client.client_id
self.client.close() # Free the handle
yield self.client.connect(client_id=client_id)
yield self.assertFailure(
self.client.get_children("/"),
NotConnectedException, ConnectionException)
test_client_session_expiration_event.timeout = 10
@inlineCallbacks
def test_client_reconnect_session_on_different_server(self):
"""On connection failure, An application can choose to use a
new connection with which to reconnect to a different member
of the zookeeper cluster, reacquiring the extant session.
A large obvious caveat to using a new client instance rather
than reconnecting the existing client, is that even though the
session has outstanding watches, the watch callbacks/deferreds
won't be active unless the client instance used to create them
is connected.
"""
session_events = []
def session_event_callback(connection, e):
session_events.append(e)
# Connect to a node in the cluster and establish a watch
self.client = ZookeeperClient(self.cluster[2].address,
session_timeout=5000)
self.client.set_session_callback(session_event_callback)
yield self.client.connect()
yield self.client.create("/hello", flags=zookeeper.EPHEMERAL)
self.assertTrue((yield self.client.exists("/hello")))
# Shutdown the server the client is connected to
self.cluster[2].stop()
yield self.sleep(0.1)
# Verify we got a session event regarding the down server
self.assertTrue(session_events)
# Open up a new connection to a different server with same session
self.client2 = ZookeeperClient(self.cluster[0].address)
yield self.client2.connect(client_id=self.client.client_id)
# Close the old disconnected client
self.client.close()
# Verify the ephemeral still exists
exists = yield self.client2.exists("/hello")
self.assertTrue(exists)
# Destroy the session and reconnect
self.client2.close()
yield self.client.connect(self.cluster[0].address)
# Ephemeral is destroyed when the session closed.
exists = yield self.client.exists("/hello")
self.assertFalse(exists)
@inlineCallbacks
def test_managed_client_backoff(self):
output = self.capture_log(level=logging.DEBUG)
self.patch(managed, 'BACKOFF_INCREMENT', 2)
self.client = yield managed.ManagedClient(
self.cluster[0].address,
connect_timeout=4).connect()
self.client2 = yield ZookeeperClient(self.cluster[1].address).connect()
exists_d, watch_d = self.client.exists_and_watch("/hello")
yield exists_d
yield self.client2.create("/hello", "world")
yield self.client2.close()
self.cluster[0].stop()
yield self.sleep(1)
# Try to do something with the connection while its down.
ops = []
ops.append(self.client.create('/abc', 'test'))
ops.append(self.client.get("/hello"))
ops.append(self.client.get_children("/"))
ops.append(self.client.set("/hello", "sad"))
# Sleep and let the session expire, and ensure we're down long enough
# for backoff to trigger.
yield self.sleep(10)
# Start the cluster and watch things work
self.cluster[0].run()
yield DeferredList(
ops, fireOnOneErrback=True, consumeErrors=True)
yield watch_d
# Verify we backed off at least once
self.assertIn("Backing off reconnect", output.getvalue())
# Verify we only reconnected once
self.assertTrue(output.getvalue().count("Restablished connection"), 1)
test_managed_client_backoff.timeout = 25
txzookeeper-0.9.8/txzookeeper/tests/mocker.py 0000664 0001750 0001750 00000243276 11745322401 021723 0 ustar kapil kapil 0000000 0000000 """
Mocker
Graceful platform for test doubles in Python: mocks, stubs, fakes, and dummies.
Copyright (c) 2007-2010, Gustavo Niemeyer
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
* Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
* Neither the name of the copyright holder nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
"""
import __builtin__
import tempfile
import unittest
import inspect
import shutil
import types
import sys
import os
import re
import gc
if sys.version_info < (2, 4):
from sets import Set as set # pragma: nocover
__all__ = ["Mocker", "Expect", "expect", "IS", "CONTAINS", "IN", "MATCH",
"ANY", "ARGS", "KWARGS", "MockerTestCase"]
__author__ = "Gustavo Niemeyer "
__license__ = "BSD"
__version__ = "1.1"
ERROR_PREFIX = "[Mocker] "
# --------------------------------------------------------------------
# Exceptions
class MatchError(AssertionError):
"""Raised when an unknown expression is seen in playback mode."""
# --------------------------------------------------------------------
# Helper for chained-style calling.
class expect(object):
"""This is a simple helper that allows a different call-style.
With this class one can comfortably do chaining of calls to the
mocker object responsible by the object being handled. For instance::
expect(obj.attr).result(3).count(1, 2)
Is the same as::
obj.attr
mocker.result(3)
mocker.count(1, 2)
"""
__mocker__ = None
def __init__(self, mock, attr=None):
self._mock = mock
self._attr = attr
def __getattr__(self, attr):
return self.__class__(self._mock, attr)
def __call__(self, *args, **kwargs):
mocker = self.__mocker__
if not mocker:
mocker = self._mock.__mocker__
getattr(mocker, self._attr)(*args, **kwargs)
return self
def Expect(mocker):
"""Create an expect() "function" using the given Mocker instance.
This helper allows defining an expect() "function" which works even
in trickier cases such as:
expect = Expect(mymocker)
expect(iter(mock)).generate([1, 2, 3])
"""
return type("Expect", (expect,), {"__mocker__": mocker})
# --------------------------------------------------------------------
# Extensions to Python's unittest.
class MockerTestCase(unittest.TestCase):
"""unittest.TestCase subclass with Mocker support.
@ivar mocker: The mocker instance.
This is a convenience only. Mocker may easily be used with the
standard C{unittest.TestCase} class if wanted.
Test methods have a Mocker instance available on C{self.mocker}.
At the end of each test method, expectations of the mocker will
be verified, and any requested changes made to the environment
will be restored.
In addition to the integration with Mocker, this class provides
a few additional helper methods.
"""
def __init__(self, methodName="runTest"):
# So here is the trick: we take the real test method, wrap it on
# a function that do the job we have to do, and insert it in the
# *instance* dictionary, so that getattr() will return our
# replacement rather than the class method.
test_method = getattr(self, methodName, None)
if test_method is not None:
def test_method_wrapper():
try:
result = test_method()
except:
raise
else:
if (self.mocker.is_recording() and
self.mocker.get_events()):
raise RuntimeError("Mocker must be put in replay "
"mode with self.mocker.replay()")
if (hasattr(result, "addCallback") and
hasattr(result, "addErrback")):
def verify(result):
self.mocker.verify()
return result
result.addCallback(verify)
else:
self.mocker.verify()
self.mocker.restore()
return result
# Copy all attributes from the original method..
for attr in dir(test_method):
# .. unless they're present in our wrapper already.
if not hasattr(test_method_wrapper, attr) or attr == "__doc__":
setattr(test_method_wrapper, attr,
getattr(test_method, attr))
setattr(self, methodName, test_method_wrapper)
# We could overload run() normally, but other well-known testing
# frameworks do it as well, and some of them won't call the super,
# which might mean that cleanup wouldn't happen. With that in mind,
# we make integration easier by using the following trick.
run_method = self.run
def run_wrapper(*args, **kwargs):
try:
return run_method(*args, **kwargs)
finally:
self.__cleanup()
self.run = run_wrapper
self.mocker = Mocker()
self.expect = Expect(self.mocker)
self.__cleanup_funcs = []
self.__cleanup_paths = []
super(MockerTestCase, self).__init__(methodName)
def __call__(self, *args, **kwargs):
# This is necessary for Python 2.3 only, because it didn't use run(),
# which is supported above.
try:
super(MockerTestCase, self).__call__(*args, **kwargs)
finally:
if sys.version_info < (2, 4):
self.__cleanup()
def __cleanup(self):
for path in self.__cleanup_paths:
if os.path.isfile(path):
os.unlink(path)
elif os.path.isdir(path):
shutil.rmtree(path)
self.mocker.reset()
for func, args, kwargs in self.__cleanup_funcs:
func(*args, **kwargs)
def addCleanup(self, func, *args, **kwargs):
self.__cleanup_funcs.append((func, args, kwargs))
def makeFile(self, content=None, suffix="", prefix="tmp", basename=None,
dirname=None, path=None):
"""Create a temporary file and return the path to it.
@param content: Initial content for the file.
@param suffix: Suffix to be given to the file's basename.
@param prefix: Prefix to be given to the file's basename.
@param basename: Full basename for the file.
@param dirname: Put file inside this directory.
The file is removed after the test runs.
"""
if path is not None:
self.__cleanup_paths.append(path)
elif basename is not None:
if dirname is None:
dirname = tempfile.mkdtemp()
self.__cleanup_paths.append(dirname)
path = os.path.join(dirname, basename)
else:
fd, path = tempfile.mkstemp(suffix, prefix, dirname)
self.__cleanup_paths.append(path)
os.close(fd)
if content is None:
os.unlink(path)
if content is not None:
file = open(path, "w")
file.write(content)
file.close()
return path
def makeDir(self, suffix="", prefix="tmp", dirname=None, path=None):
"""Create a temporary directory and return the path to it.
@param suffix: Suffix to be given to the file's basename.
@param prefix: Prefix to be given to the file's basename.
@param dirname: Put directory inside this parent directory.
The directory is removed after the test runs.
"""
if path is not None:
os.makedirs(path)
else:
path = tempfile.mkdtemp(suffix, prefix, dirname)
self.__cleanup_paths.append(path)
return path
def failUnlessIs(self, first, second, msg=None):
"""Assert that C{first} is the same object as C{second}."""
if first is not second:
raise self.failureException(msg or "%r is not %r" % (first, second))
def failIfIs(self, first, second, msg=None):
"""Assert that C{first} is not the same object as C{second}."""
if first is second:
raise self.failureException(msg or "%r is %r" % (first, second))
def failUnlessIn(self, first, second, msg=None):
"""Assert that C{first} is contained in C{second}."""
if first not in second:
raise self.failureException(msg or "%r not in %r" % (first, second))
def failUnlessStartsWith(self, first, second, msg=None):
"""Assert that C{first} starts with C{second}."""
if first[:len(second)] != second:
raise self.failureException(msg or "%r doesn't start with %r" %
(first, second))
def failIfStartsWith(self, first, second, msg=None):
"""Assert that C{first} doesn't start with C{second}."""
if first[:len(second)] == second:
raise self.failureException(msg or "%r starts with %r" %
(first, second))
def failUnlessEndsWith(self, first, second, msg=None):
"""Assert that C{first} starts with C{second}."""
if first[len(first)-len(second):] != second:
raise self.failureException(msg or "%r doesn't end with %r" %
(first, second))
def failIfEndsWith(self, first, second, msg=None):
"""Assert that C{first} doesn't start with C{second}."""
if first[len(first)-len(second):] == second:
raise self.failureException(msg or "%r ends with %r" %
(first, second))
def failIfIn(self, first, second, msg=None):
"""Assert that C{first} is not contained in C{second}."""
if first in second:
raise self.failureException(msg or "%r in %r" % (first, second))
def failUnlessApproximates(self, first, second, tolerance, msg=None):
"""Assert that C{first} is near C{second} by at most C{tolerance}."""
if abs(first - second) > tolerance:
raise self.failureException(msg or "abs(%r - %r) > %r" %
(first, second, tolerance))
def failIfApproximates(self, first, second, tolerance, msg=None):
"""Assert that C{first} is far from C{second} by at least C{tolerance}.
"""
if abs(first - second) <= tolerance:
raise self.failureException(msg or "abs(%r - %r) <= %r" %
(first, second, tolerance))
def failUnlessMethodsMatch(self, first, second):
"""Assert that public methods in C{first} are present in C{second}.
This method asserts that all public methods found in C{first} are also
present in C{second} and accept the same arguments. C{first} may
have its own private methods, though, and may not have all methods
found in C{second}. Note that if a private method in C{first} matches
the name of one in C{second}, their specification is still compared.
This is useful to verify if a fake or stub class have the same API as
the real class being simulated.
"""
first_methods = dict(inspect.getmembers(first, inspect.ismethod))
second_methods = dict(inspect.getmembers(second, inspect.ismethod))
for name, first_method in first_methods.iteritems():
first_argspec = inspect.getargspec(first_method)
first_formatted = inspect.formatargspec(*first_argspec)
second_method = second_methods.get(name)
if second_method is None:
if name[:1] == "_":
continue # First may have its own private methods.
raise self.failureException("%s.%s%s not present in %s" %
(first.__name__, name, first_formatted, second.__name__))
second_argspec = inspect.getargspec(second_method)
if first_argspec != second_argspec:
second_formatted = inspect.formatargspec(*second_argspec)
raise self.failureException("%s.%s%s != %s.%s%s" %
(first.__name__, name, first_formatted,
second.__name__, name, second_formatted))
def failUnlessRaises(self, excClass, *args, **kwargs):
"""
Fail unless an exception of class excClass is thrown by callableObj
when invoked with arguments args and keyword arguments kwargs. If a
different type of exception is thrown, it will not be caught, and the
test case will be deemed to have suffered an error, exactly as for an
unexpected exception. It returns the exception instance if it matches
the given exception class.
This may also be used as a context manager when provided with a single
argument, as such:
with self.failUnlessRaises(ExcClass):
logic_which_should_raise()
"""
return self.failUnlessRaisesRegexp(excClass, None, *args, **kwargs)
def failUnlessRaisesRegexp(self, excClass, regexp, *args, **kwargs):
"""
Fail unless an exception of class excClass is thrown by callableObj
when invoked with arguments args and keyword arguments kwargs, and
the str(error) value matches the provided regexp. If a different type
of exception is thrown, it will not be caught, and the test case will
be deemed to have suffered an error, exactly as for an unexpected
exception. It returns the exception instance if it matches the given
exception class.
This may also be used as a context manager when provided with a single
argument, as such:
with self.failUnlessRaisesRegexp(ExcClass, "something like.*happened"):
logic_which_should_raise()
"""
def match_regexp(error):
error_str = str(error)
if regexp is not None and not re.search(regexp, error_str):
raise self.failureException("%r doesn't match %r" %
(error_str, regexp))
excName = self.__class_name(excClass)
if args:
callableObj = args[0]
try:
result = callableObj(*args[1:], **kwargs)
except excClass, e:
match_regexp(e)
return e
else:
raise self.failureException("%s not raised (%r returned)" %
(excName, result))
else:
test = self
class AssertRaisesContextManager(object):
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.exception = value
if value is None:
raise test.failureException("%s not raised" % excName)
elif isinstance(value, excClass):
match_regexp(value)
return True
return AssertRaisesContextManager()
def __class_name(self, cls):
return getattr(cls, "__name__", str(cls))
def failUnlessIsInstance(self, obj, cls, msg=None):
"""Assert that isinstance(obj, cls)."""
if not isinstance(obj, cls):
if msg is None:
msg = "%r is not an instance of %s" % \
(obj, self.__class_name(cls))
raise self.failureException(msg)
def failIfIsInstance(self, obj, cls, msg=None):
"""Assert that isinstance(obj, cls) is False."""
if isinstance(obj, cls):
if msg is None:
msg = "%r is an instance of %s" % \
(obj, self.__class_name(cls))
raise self.failureException(msg)
assertIs = failUnlessIs
assertIsNot = failIfIs
assertIn = failUnlessIn
assertNotIn = failIfIn
assertStartsWith = failUnlessStartsWith
assertNotStartsWith = failIfStartsWith
assertEndsWith = failUnlessEndsWith
assertNotEndsWith = failIfEndsWith
assertApproximates = failUnlessApproximates
assertNotApproximates = failIfApproximates
assertMethodsMatch = failUnlessMethodsMatch
assertRaises = failUnlessRaises
assertRaisesRegexp = failUnlessRaisesRegexp
assertIsInstance = failUnlessIsInstance
assertIsNotInstance = failIfIsInstance
assertNotIsInstance = failIfIsInstance # Poor choice in 2.7/3.2+.
# The following are missing in Python < 2.4.
assertTrue = unittest.TestCase.failUnless
assertFalse = unittest.TestCase.failIf
# The following is provided for compatibility with Twisted's trial.
assertIdentical = assertIs
assertNotIdentical = assertIsNot
failUnlessIdentical = failUnlessIs
failIfIdentical = failIfIs
# --------------------------------------------------------------------
# Mocker.
class classinstancemethod(object):
def __init__(self, method):
self.method = method
def __get__(self, obj, cls=None):
def bound_method(*args, **kwargs):
return self.method(cls, obj, *args, **kwargs)
return bound_method
class MockerBase(object):
"""Controller of mock objects.
A mocker instance is used to command recording and replay of
expectations on any number of mock objects.
Expectations should be expressed for the mock object while in
record mode (the initial one) by using the mock object itself,
and using the mocker (and/or C{expect()} as a helper) to define
additional behavior for each event. For instance::
mock = mocker.mock()
mock.hello()
mocker.result("Hi!")
mocker.replay()
assert mock.hello() == "Hi!"
mock.restore()
mock.verify()
In this short excerpt a mock object is being created, then an
expectation of a call to the C{hello()} method was recorded, and
when called the method should return the value C{10}. Then, the
mocker is put in replay mode, and the expectation is satisfied by
calling the C{hello()} method, which indeed returns 10. Finally,
a call to the L{restore()} method is performed to undo any needed
changes made in the environment, and the L{verify()} method is
called to ensure that all defined expectations were met.
The same logic can be expressed more elegantly using the
C{with mocker:} statement, as follows::
mock = mocker.mock()
mock.hello()
mocker.result("Hi!")
with mocker:
assert mock.hello() == "Hi!"
Also, the MockerTestCase class, which integrates the mocker on
a unittest.TestCase subclass, may be used to reduce the overhead
of controlling the mocker. A test could be written as follows::
class SampleTest(MockerTestCase):
def test_hello(self):
mock = self.mocker.mock()
mock.hello()
self.mocker.result("Hi!")
self.mocker.replay()
self.assertEquals(mock.hello(), "Hi!")
"""
_recorders = []
# For convenience only.
on = expect
class __metaclass__(type):
def __init__(self, name, bases, dict):
# Make independent lists on each subclass, inheriting from parent.
self._recorders = list(getattr(self, "_recorders", ()))
def __init__(self):
self._recorders = self._recorders[:]
self._events = []
self._recording = True
self._ordering = False
self._last_orderer = None
def is_recording(self):
"""Return True if in recording mode, False if in replay mode.
Recording is the initial state.
"""
return self._recording
def replay(self):
"""Change to replay mode, where recorded events are reproduced.
If already in replay mode, the mocker will be restored, with all
expectations reset, and then put again in replay mode.
An alternative and more comfortable way to replay changes is
using the 'with' statement, as follows::
mocker = Mocker()
with mocker:
The 'with' statement will automatically put mocker in replay
mode, and will also verify if all events were correctly reproduced
at the end (using L{verify()}), and also restore any changes done
in the environment (with L{restore()}).
Also check the MockerTestCase class, which integrates the
unittest.TestCase class with mocker.
"""
if not self._recording:
for event in self._events:
event.restore()
else:
self._recording = False
for event in self._events:
event.replay()
def restore(self):
"""Restore changes in the environment, and return to recording mode.
This should always be called after the test is complete (succeeding
or not). There are ways to call this method automatically on
completion (e.g. using a C{with mocker:} statement, or using the
L{MockerTestCase} class.
"""
if not self._recording:
self._recording = True
for event in self._events:
event.restore()
def reset(self):
"""Reset the mocker state.
This will restore environment changes, if currently in replay
mode, and then remove all events previously recorded.
"""
if not self._recording:
self.restore()
self.unorder()
del self._events[:]
def get_events(self):
"""Return all recorded events."""
return self._events[:]
def add_event(self, event):
"""Add an event.
This method is used internally by the implementation, and
shouldn't be needed on normal mocker usage.
"""
self._events.append(event)
if self._ordering:
orderer = event.add_task(Orderer(event.path))
if self._last_orderer:
orderer.add_dependency(self._last_orderer)
self._last_orderer = orderer
return event
def verify(self):
"""Check if all expectations were met, and raise AssertionError if not.
The exception message will include a nice description of which
expectations were not met, and why.
"""
errors = []
for event in self._events:
try:
event.verify()
except AssertionError, e:
error = str(e)
if not error:
raise RuntimeError("Empty error message from %r"
% event)
errors.append(error)
if errors:
message = [ERROR_PREFIX + "Unmet expectations:", ""]
for error in errors:
lines = error.splitlines()
message.append("=> " + lines.pop(0))
message.extend([" " + line for line in lines])
message.append("")
raise AssertionError(os.linesep.join(message))
def mock(self, spec_and_type=None, spec=None, type=None,
name=None, count=True):
"""Return a new mock object.
@param spec_and_type: Handy positional argument which sets both
spec and type.
@param spec: Method calls will be checked for correctness against
the given class.
@param type: If set, the Mock's __class__ attribute will return
the given type. This will make C{isinstance()} calls
on the object work.
@param name: Name for the mock object, used in the representation of
expressions. The name is rarely needed, as it's usually
guessed correctly from the variable name used.
@param count: If set to false, expressions may be executed any number
of times, unless an expectation is explicitly set using
the L{count()} method. By default, expressions are
expected once.
"""
if spec_and_type is not None:
spec = type = spec_and_type
return Mock(self, spec=spec, type=type, name=name, count=count)
def proxy(self, object, spec=True, type=True, name=None, count=True,
passthrough=True):
"""Return a new mock object which proxies to the given object.
Proxies are useful when only part of the behavior of an object
is to be mocked. Unknown expressions may be passed through to
the real implementation implicitly (if the C{passthrough} argument
is True), or explicitly (using the L{passthrough()} method
on the event).
@param object: Real object to be proxied, and replaced by the mock
on replay mode. It may also be an "import path",
such as C{"time.time"}, in which case the object
will be the C{time} function from the C{time} module.
@param spec: Method calls will be checked for correctness against
the given object, which may be a class or an instance
where attributes will be looked up. Defaults to the
the C{object} parameter. May be set to None explicitly,
in which case spec checking is disabled. Checks may
also be disabled explicitly on a per-event basis with
the L{nospec()} method.
@param type: If set, the Mock's __class__ attribute will return
the given type. This will make C{isinstance()} calls
on the object work. Defaults to the type of the
C{object} parameter. May be set to None explicitly.
@param name: Name for the mock object, used in the representation of
expressions. The name is rarely needed, as it's usually
guessed correctly from the variable name used.
@param count: If set to false, expressions may be executed any number
of times, unless an expectation is explicitly set using
the L{count()} method. By default, expressions are
expected once.
@param passthrough: If set to False, passthrough of actions on the
proxy to the real object will only happen when
explicitly requested via the L{passthrough()}
method.
"""
if isinstance(object, basestring):
if name is None:
name = object
import_stack = object.split(".")
attr_stack = []
while import_stack:
module_path = ".".join(import_stack)
try:
__import__(module_path)
except ImportError:
attr_stack.insert(0, import_stack.pop())
if not import_stack:
raise
continue
else:
object = sys.modules[module_path]
for attr in attr_stack:
object = getattr(object, attr)
break
if isinstance(object, types.UnboundMethodType):
object = object.im_func
if spec is True:
spec = object
if type is True:
type = __builtin__.type(object)
return Mock(self, spec=spec, type=type, object=object,
name=name, count=count, passthrough=passthrough)
def replace(self, object, spec=True, type=True, name=None, count=True,
passthrough=True):
"""Create a proxy, and replace the original object with the mock.
On replay, the original object will be replaced by the returned
proxy in all dictionaries found in the running interpreter via
the garbage collecting system. This should cover module
namespaces, class namespaces, instance namespaces, and so on.
@param object: Real object to be proxied, and replaced by the mock
on replay mode. It may also be an "import path",
such as C{"time.time"}, in which case the object
will be the C{time} function from the C{time} module.
@param spec: Method calls will be checked for correctness against
the given object, which may be a class or an instance
where attributes will be looked up. Defaults to the
the C{object} parameter. May be set to None explicitly,
in which case spec checking is disabled. Checks may
also be disabled explicitly on a per-event basis with
the L{nospec()} method.
@param type: If set, the Mock's __class__ attribute will return
the given type. This will make C{isinstance()} calls
on the object work. Defaults to the type of the
C{object} parameter. May be set to None explicitly.
@param name: Name for the mock object, used in the representation of
expressions. The name is rarely needed, as it's usually
guessed correctly from the variable name used.
@param passthrough: If set to False, passthrough of actions on the
proxy to the real object will only happen when
explicitly requested via the L{passthrough()}
method.
"""
mock = self.proxy(object, spec, type, name, count, passthrough)
event = self._get_replay_restore_event()
event.add_task(ProxyReplacer(mock))
return mock
def patch(self, object, spec=True):
"""Patch an existing object to reproduce recorded events.
@param object: Class or instance to be patched.
@param spec: Method calls will be checked for correctness against
the given object, which may be a class or an instance
where attributes will be looked up. Defaults to the
the C{object} parameter. May be set to None explicitly,
in which case spec checking is disabled. Checks may
also be disabled explicitly on a per-event basis with
the L{nospec()} method.
The result of this method is still a mock object, which can be
used like any other mock object to record events. The difference
is that when the mocker is put on replay mode, the *real* object
will be modified to behave according to recorded expectations.
Patching works in individual instances, and also in classes.
When an instance is patched, recorded events will only be
considered on this specific instance, and other instances should
behave normally. When a class is patched, the reproduction of
events will be considered on any instance of this class once
created (collectively).
Observe that, unlike with proxies which catch only events done
through the mock object, *all* accesses to recorded expectations
will be considered; even these coming from the object itself
(e.g. C{self.hello()} is considered if this method was patched).
While this is a very powerful feature, and many times the reason
to use patches in the first place, it's important to keep this
behavior in mind.
Patching of the original object only takes place when the mocker
is put on replay mode, and the patched object will be restored
to its original state once the L{restore()} method is called
(explicitly, or implicitly with alternative conventions, such as
a C{with mocker:} block, or a MockerTestCase class).
"""
if spec is True:
spec = object
patcher = Patcher()
event = self._get_replay_restore_event()
event.add_task(patcher)
mock = Mock(self, object=object, patcher=patcher,
passthrough=True, spec=spec)
patcher.patch_attr(object, '__mocker_mock__', mock)
return mock
def act(self, path):
"""This is called by mock objects whenever something happens to them.
This method is part of the interface between the mocker
and mock objects.
"""
if self._recording:
event = self.add_event(Event(path))
for recorder in self._recorders:
recorder(self, event)
return Mock(self, path)
else:
# First run events that may run, then run unsatisfied events, then
# ones not previously run. We put the index in the ordering tuple
# instead of the actual event because we want a stable sort
# (ordering between 2 events is undefined).
events = self._events
order = [(events[i].satisfied()*2 + events[i].has_run(), i)
for i in range(len(events))]
order.sort()
postponed = None
for weight, i in order:
event = events[i]
if event.matches(path):
if event.may_run(path):
return event.run(path)
elif postponed is None:
postponed = event
if postponed is not None:
return postponed.run(path)
raise MatchError(ERROR_PREFIX + "Unexpected expression: %s" % path)
def get_recorders(cls, self):
"""Return recorders associated with this mocker class or instance.
This method may be called on mocker instances and also on mocker
classes. See the L{add_recorder()} method for more information.
"""
return (self or cls)._recorders[:]
get_recorders = classinstancemethod(get_recorders)
def add_recorder(cls, self, recorder):
"""Add a recorder to this mocker class or instance.
@param recorder: Callable accepting C{(mocker, event)} as parameters.
This is part of the implementation of mocker.
All registered recorders are called for translating events that
happen during recording into expectations to be met once the state
is switched to replay mode.
This method may be called on mocker instances and also on mocker
classes. When called on a class, the recorder will be used by
all instances, and also inherited on subclassing. When called on
instances, the recorder is added only to the given instance.
"""
(self or cls)._recorders.append(recorder)
return recorder
add_recorder = classinstancemethod(add_recorder)
def remove_recorder(cls, self, recorder):
"""Remove the given recorder from this mocker class or instance.
This method may be called on mocker classes and also on mocker
instances. See the L{add_recorder()} method for more information.
"""
(self or cls)._recorders.remove(recorder)
remove_recorder = classinstancemethod(remove_recorder)
def result(self, value):
"""Make the last recorded event return the given value on replay.
@param value: Object to be returned when the event is replayed.
"""
self.call(lambda *args, **kwargs: value)
def generate(self, sequence):
"""Last recorded event will return a generator with the given sequence.
@param sequence: Sequence of values to be generated.
"""
def generate(*args, **kwargs):
for value in sequence:
yield value
self.call(generate)
def throw(self, exception):
"""Make the last recorded event raise the given exception on replay.
@param exception: Class or instance of exception to be raised.
"""
def raise_exception(*args, **kwargs):
raise exception
self.call(raise_exception)
def call(self, func, with_object=False):
"""Make the last recorded event cause the given function to be called.
@param func: Function to be called.
@param with_object: If True, the called function will receive the
patched or proxied object so that its state may be used or verified
in checks.
The result of the function will be used as the event result.
"""
event = self._events[-1]
if with_object and event.path.root_object is None:
raise TypeError("Mock object isn't a proxy")
event.add_task(FunctionRunner(func, with_root_object=with_object))
def count(self, min, max=False):
"""Last recorded event must be replayed between min and max times.
@param min: Minimum number of times that the event must happen.
@param max: Maximum number of times that the event must happen. If
not given, it defaults to the same value of the C{min}
parameter. If set to None, there is no upper limit, and
the expectation is met as long as it happens at least
C{min} times.
"""
event = self._events[-1]
for task in event.get_tasks():
if isinstance(task, RunCounter):
event.remove_task(task)
event.prepend_task(RunCounter(min, max))
def is_ordering(self):
"""Return true if all events are being ordered.
See the L{order()} method.
"""
return self._ordering
def unorder(self):
"""Disable the ordered mode.
See the L{order()} method for more information.
"""
self._ordering = False
self._last_orderer = None
def order(self, *path_holders):
"""Create an expectation of order between two or more events.
@param path_holders: Objects returned as the result of recorded events.
By default, mocker won't force events to happen precisely in
the order they were recorded. Calling this method will change
this behavior so that events will only match if reproduced in
the correct order.
There are two ways in which this method may be used. Which one
is used in a given occasion depends only on convenience.
If no arguments are passed, the mocker will be put in a mode where
all the recorded events following the method call will only be met
if they happen in order. When that's used, the mocker may be put
back in unordered mode by calling the L{unorder()} method, or by
using a 'with' block, like so::
with mocker.ordered():
In this case, only expressions in will be ordered,
and the mocker will be back in unordered mode after the 'with' block.
The second way to use it is by specifying precisely which events
should be ordered. As an example::
mock = mocker.mock()
expr1 = mock.hello()
expr2 = mock.world
expr3 = mock.x.y.z
mocker.order(expr1, expr2, expr3)
This method of ordering only works when the expression returns
another object.
Also check the L{after()} and L{before()} methods, which are
alternative ways to perform this.
"""
if not path_holders:
self._ordering = True
return OrderedContext(self)
last_orderer = None
for path_holder in path_holders:
if type(path_holder) is Path:
path = path_holder
else:
path = path_holder.__mocker_path__
for event in self._events:
if event.path is path:
for task in event.get_tasks():
if isinstance(task, Orderer):
orderer = task
break
else:
orderer = Orderer(path)
event.add_task(orderer)
if last_orderer:
orderer.add_dependency(last_orderer)
last_orderer = orderer
break
def after(self, *path_holders):
"""Last recorded event must happen after events referred to.
@param path_holders: Objects returned as the result of recorded events
which should happen before the last recorded event
As an example, the idiom::
expect(mock.x).after(mock.y, mock.z)
is an alternative way to say::
expr_x = mock.x
expr_y = mock.y
expr_z = mock.z
mocker.order(expr_y, expr_x)
mocker.order(expr_z, expr_x)
See L{order()} for more information.
"""
last_path = self._events[-1].path
for path_holder in path_holders:
self.order(path_holder, last_path)
def before(self, *path_holders):
"""Last recorded event must happen before events referred to.
@param path_holders: Objects returned as the result of recorded events
which should happen after the last recorded event
As an example, the idiom::
expect(mock.x).before(mock.y, mock.z)
is an alternative way to say::
expr_x = mock.x
expr_y = mock.y
expr_z = mock.z
mocker.order(expr_x, expr_y)
mocker.order(expr_x, expr_z)
See L{order()} for more information.
"""
last_path = self._events[-1].path
for path_holder in path_holders:
self.order(last_path, path_holder)
def nospec(self):
"""Don't check method specification of real object on last event.
By default, when using a mock created as the result of a call to
L{proxy()}, L{replace()}, and C{patch()}, or when passing the spec
attribute to the L{mock()} method, method calls on the given object
are checked for correctness against the specification of the real
object (or the explicitly provided spec).
This method will disable that check specifically for the last
recorded event.
"""
event = self._events[-1]
for task in event.get_tasks():
if isinstance(task, SpecChecker):
event.remove_task(task)
def passthrough(self, result_callback=None):
"""Make the last recorded event run on the real object once seen.
@param result_callback: If given, this function will be called with
the result of the *real* method call as the only argument.
This can only be used on proxies, as returned by the L{proxy()}
and L{replace()} methods, or on mocks representing patched objects,
as returned by the L{patch()} method.
"""
event = self._events[-1]
if event.path.root_object is None:
raise TypeError("Mock object isn't a proxy")
event.add_task(PathExecuter(result_callback))
def __enter__(self):
"""Enter in a 'with' context. This will run replay()."""
self.replay()
return self
def __exit__(self, type, value, traceback):
"""Exit from a 'with' context.
This will run restore() at all times, but will only run verify()
if the 'with' block itself hasn't raised an exception. Exceptions
in that block are never swallowed.
"""
self.restore()
if type is None:
self.verify()
return False
def _get_replay_restore_event(self):
"""Return unique L{ReplayRestoreEvent}, creating if needed.
Some tasks only want to replay/restore. When that's the case,
they shouldn't act on other events during replay. Also, they
can all be put in a single event when that's the case. Thus,
we add a single L{ReplayRestoreEvent} as the first element of
the list.
"""
if not self._events or type(self._events[0]) != ReplayRestoreEvent:
self._events.insert(0, ReplayRestoreEvent())
return self._events[0]
class OrderedContext(object):
def __init__(self, mocker):
self._mocker = mocker
def __enter__(self):
return None
def __exit__(self, type, value, traceback):
self._mocker.unorder()
class Mocker(MockerBase):
__doc__ = MockerBase.__doc__
# Decorator to add recorders on the standard Mocker class.
recorder = Mocker.add_recorder
# --------------------------------------------------------------------
# Mock object.
class Mock(object):
def __init__(self, mocker, path=None, name=None, spec=None, type=None,
object=None, passthrough=False, patcher=None, count=True):
self.__mocker__ = mocker
self.__mocker_path__ = path or Path(self, object)
self.__mocker_name__ = name
self.__mocker_spec__ = spec
self.__mocker_object__ = object
self.__mocker_passthrough__ = passthrough
self.__mocker_patcher__ = patcher
self.__mocker_replace__ = False
self.__mocker_type__ = type
self.__mocker_count__ = count
def __mocker_act__(self, kind, args=(), kwargs={}, object=None):
if self.__mocker_name__ is None:
self.__mocker_name__ = find_object_name(self, 2)
action = Action(kind, args, kwargs, self.__mocker_path__)
path = self.__mocker_path__ + action
if object is not None:
path.root_object = object
try:
return self.__mocker__.act(path)
except MatchError, exception:
root_mock = path.root_mock
if (path.root_object is not None and
root_mock.__mocker_passthrough__):
return path.execute(path.root_object)
# Reinstantiate to show raise statement on traceback, and
# also to make the traceback shown shorter.
raise MatchError(str(exception))
except AssertionError, e:
lines = str(e).splitlines()
message = [ERROR_PREFIX + "Unmet expectation:", ""]
message.append("=> " + lines.pop(0))
message.extend([" " + line for line in lines])
message.append("")
raise AssertionError(os.linesep.join(message))
def __getattribute__(self, name):
if name.startswith("__mocker_"):
return super(Mock, self).__getattribute__(name)
if name == "__class__":
if self.__mocker__.is_recording() or self.__mocker_type__ is None:
return type(self)
return self.__mocker_type__
if name == "__length_hint__":
# This is used by Python 2.6+ to optimize the allocation
# of arrays in certain cases. Pretend it doesn't exist.
raise AttributeError("No __length_hint__ here!")
return self.__mocker_act__("getattr", (name,))
def __setattr__(self, name, value):
if name.startswith("__mocker_"):
return super(Mock, self).__setattr__(name, value)
return self.__mocker_act__("setattr", (name, value))
def __delattr__(self, name):
return self.__mocker_act__("delattr", (name,))
def __call__(self, *args, **kwargs):
return self.__mocker_act__("call", args, kwargs)
def __contains__(self, value):
return self.__mocker_act__("contains", (value,))
def __getitem__(self, key):
return self.__mocker_act__("getitem", (key,))
def __setitem__(self, key, value):
return self.__mocker_act__("setitem", (key, value))
def __delitem__(self, key):
return self.__mocker_act__("delitem", (key,))
def __len__(self):
# MatchError is turned on an AttributeError so that list() and
# friends act properly when trying to get length hints on
# something that doesn't offer them.
try:
result = self.__mocker_act__("len")
except MatchError, e:
raise AttributeError(str(e))
if type(result) is Mock:
return 0
return result
def __nonzero__(self):
try:
result = self.__mocker_act__("nonzero")
except MatchError, e:
return True
if type(result) is Mock:
return True
return result
def __iter__(self):
# XXX On py3k, when next() becomes __next__(), we'll be able
# to return the mock itself because it will be considered
# an iterator (we'll be mocking __next__ as well, which we
# can't now).
result = self.__mocker_act__("iter")
if type(result) is Mock:
return iter([])
return result
# When adding a new action kind here, also add support for it on
# Action.execute() and Path.__str__().
def find_object_name(obj, depth=0):
"""Try to detect how the object is named on a previous scope."""
try:
frame = sys._getframe(depth+1)
except:
return None
for name, frame_obj in frame.f_locals.iteritems():
if frame_obj is obj:
return name
self = frame.f_locals.get("self")
if self is not None:
try:
items = list(self.__dict__.iteritems())
except:
pass
else:
for name, self_obj in items:
if self_obj is obj:
return name
return None
# --------------------------------------------------------------------
# Action and path.
class Action(object):
def __init__(self, kind, args, kwargs, path=None):
self.kind = kind
self.args = args
self.kwargs = kwargs
self.path = path
self._execute_cache = {}
def __repr__(self):
if self.path is None:
return "Action(%r, %r, %r)" % (self.kind, self.args, self.kwargs)
return "Action(%r, %r, %r, %r)" % \
(self.kind, self.args, self.kwargs, self.path)
def __eq__(self, other):
return (self.kind == other.kind and
self.args == other.args and
self.kwargs == other.kwargs)
def __ne__(self, other):
return not self.__eq__(other)
def matches(self, other):
return (self.kind == other.kind and
match_params(self.args, self.kwargs, other.args, other.kwargs))
def execute(self, object):
# This caching scheme may fail if the object gets deallocated before
# the action, as the id might get reused. It's somewhat easy to fix
# that with a weakref callback. For our uses, though, the object
# should never get deallocated before the action itself, so we'll
# just keep it simple.
if id(object) in self._execute_cache:
return self._execute_cache[id(object)]
execute = getattr(object, "__mocker_execute__", None)
if execute is not None:
result = execute(self, object)
else:
kind = self.kind
if kind == "getattr":
result = getattr(object, self.args[0])
elif kind == "setattr":
result = setattr(object, self.args[0], self.args[1])
elif kind == "delattr":
result = delattr(object, self.args[0])
elif kind == "call":
result = object(*self.args, **self.kwargs)
elif kind == "contains":
result = self.args[0] in object
elif kind == "getitem":
result = object[self.args[0]]
elif kind == "setitem":
result = object[self.args[0]] = self.args[1]
elif kind == "delitem":
del object[self.args[0]]
result = None
elif kind == "len":
result = len(object)
elif kind == "nonzero":
result = bool(object)
elif kind == "iter":
result = iter(object)
else:
raise RuntimeError("Don't know how to execute %r kind." % kind)
self._execute_cache[id(object)] = result
return result
class Path(object):
def __init__(self, root_mock, root_object=None, actions=()):
self.root_mock = root_mock
self.root_object = root_object
self.actions = tuple(actions)
self.__mocker_replace__ = False
def parent_path(self):
if not self.actions:
return None
return self.actions[-1].path
parent_path = property(parent_path)
def __add__(self, action):
"""Return a new path which includes the given action at the end."""
return self.__class__(self.root_mock, self.root_object,
self.actions + (action,))
def __eq__(self, other):
"""Verify if the two paths are equal.
Two paths are equal if they refer to the same mock object, and
have the actions with equal kind, args and kwargs.
"""
if (self.root_mock is not other.root_mock or
self.root_object is not other.root_object or
len(self.actions) != len(other.actions)):
return False
for action, other_action in zip(self.actions, other.actions):
if action != other_action:
return False
return True
def matches(self, other):
"""Verify if the two paths are equivalent.
Two paths are equal if they refer to the same mock object, and
have the same actions performed on them.
"""
if (self.root_mock is not other.root_mock or
len(self.actions) != len(other.actions)):
return False
for action, other_action in zip(self.actions, other.actions):
if not action.matches(other_action):
return False
return True
def execute(self, object):
"""Execute all actions sequentially on object, and return result.
"""
for action in self.actions:
object = action.execute(object)
return object
def __str__(self):
"""Transform the path into a nice string such as obj.x.y('z')."""
result = self.root_mock.__mocker_name__ or ""
for action in self.actions:
if action.kind == "getattr":
result = "%s.%s" % (result, action.args[0])
elif action.kind == "setattr":
result = "%s.%s = %r" % (result, action.args[0], action.args[1])
elif action.kind == "delattr":
result = "del %s.%s" % (result, action.args[0])
elif action.kind == "call":
args = [repr(x) for x in action.args]
items = list(action.kwargs.iteritems())
items.sort()
for pair in items:
args.append("%s=%r" % pair)
result = "%s(%s)" % (result, ", ".join(args))
elif action.kind == "contains":
result = "%r in %s" % (action.args[0], result)
elif action.kind == "getitem":
result = "%s[%r]" % (result, action.args[0])
elif action.kind == "setitem":
result = "%s[%r] = %r" % (result, action.args[0],
action.args[1])
elif action.kind == "delitem":
result = "del %s[%r]" % (result, action.args[0])
elif action.kind == "len":
result = "len(%s)" % result
elif action.kind == "nonzero":
result = "bool(%s)" % result
elif action.kind == "iter":
result = "iter(%s)" % result
else:
raise RuntimeError("Don't know how to format kind %r" %
action.kind)
return result
class SpecialArgument(object):
"""Base for special arguments for matching parameters."""
def __init__(self, object=None):
self.object = object
def __repr__(self):
if self.object is None:
return self.__class__.__name__
else:
return "%s(%r)" % (self.__class__.__name__, self.object)
def matches(self, other):
return True
def __eq__(self, other):
return type(other) == type(self) and self.object == other.object
class ANY(SpecialArgument):
"""Matches any single argument."""
ANY = ANY()
class ARGS(SpecialArgument):
"""Matches zero or more positional arguments."""
ARGS = ARGS()
class KWARGS(SpecialArgument):
"""Matches zero or more keyword arguments."""
KWARGS = KWARGS()
class IS(SpecialArgument):
def matches(self, other):
return self.object is other
def __eq__(self, other):
return type(other) == type(self) and self.object is other.object
class CONTAINS(SpecialArgument):
def matches(self, other):
try:
other.__contains__
except AttributeError:
try:
iter(other)
except TypeError:
# If an object can't be iterated, and has no __contains__
# hook, it'd blow up on the test below. We test this in
# advance to prevent catching more errors than we really
# want.
return False
return self.object in other
class IN(SpecialArgument):
def matches(self, other):
return other in self.object
class MATCH(SpecialArgument):
def matches(self, other):
return bool(self.object(other))
def __eq__(self, other):
return type(other) == type(self) and self.object is other.object
def match_params(args1, kwargs1, args2, kwargs2):
"""Match the two sets of parameters, considering special parameters."""
has_args = ARGS in args1
has_kwargs = KWARGS in args1
if has_kwargs:
args1 = [arg1 for arg1 in args1 if arg1 is not KWARGS]
elif len(kwargs1) != len(kwargs2):
return False
if not has_args and len(args1) != len(args2):
return False
# Either we have the same number of kwargs, or unknown keywords are
# accepted (KWARGS was used), so check just the ones in kwargs1.
for key, arg1 in kwargs1.iteritems():
if key not in kwargs2:
return False
arg2 = kwargs2[key]
if isinstance(arg1, SpecialArgument):
if not arg1.matches(arg2):
return False
elif arg1 != arg2:
return False
# Keywords match. Now either we have the same number of
# arguments, or ARGS was used. If ARGS wasn't used, arguments
# must match one-on-one necessarily.
if not has_args:
for arg1, arg2 in zip(args1, args2):
if isinstance(arg1, SpecialArgument):
if not arg1.matches(arg2):
return False
elif arg1 != arg2:
return False
return True
# Easy choice. Keywords are matching, and anything on args is accepted.
if (ARGS,) == args1:
return True
# We have something different there. If we don't have positional
# arguments on the original call, it can't match.
if not args2:
# Unless we have just several ARGS (which is bizarre, but..).
for arg1 in args1:
if arg1 is not ARGS:
return False
return True
# Ok, all bets are lost. We have to actually do the more expensive
# matching. This is an algorithm based on the idea of the Levenshtein
# Distance between two strings, but heavily hacked for this purpose.
args2l = len(args2)
if args1[0] is ARGS:
args1 = args1[1:]
array = [0]*args2l
else:
array = [1]*args2l
for i in range(len(args1)):
last = array[0]
if args1[i] is ARGS:
for j in range(1, args2l):
last, array[j] = array[j], min(array[j-1], array[j], last)
else:
array[0] = i or int(args1[i] != args2[0])
for j in range(1, args2l):
last, array[j] = array[j], last or int(args1[i] != args2[j])
if 0 not in array:
return False
if array[-1] != 0:
return False
return True
# --------------------------------------------------------------------
# Event and task base.
class Event(object):
"""Aggregation of tasks that keep track of a recorded action.
An event represents something that may or may not happen while the
mocked environment is running, such as an attribute access, or a
method call. The event is composed of several tasks that are
orchestrated together to create a composed meaning for the event,
including for which actions it should be run, what happens when it
runs, and what's the expectations about the actions run.
"""
def __init__(self, path=None):
self.path = path
self._tasks = []
self._has_run = False
def add_task(self, task):
"""Add a new task to this task."""
self._tasks.append(task)
return task
def prepend_task(self, task):
"""Add a task at the front of the list."""
self._tasks.insert(0, task)
return task
def remove_task(self, task):
self._tasks.remove(task)
def replace_task(self, old_task, new_task):
"""Replace old_task with new_task, in the same position."""
for i in range(len(self._tasks)):
if self._tasks[i] is old_task:
self._tasks[i] = new_task
return new_task
def get_tasks(self):
return self._tasks[:]
def matches(self, path):
"""Return true if *all* tasks match the given path."""
for task in self._tasks:
if not task.matches(path):
return False
return bool(self._tasks)
def has_run(self):
return self._has_run
def may_run(self, path):
"""Verify if any task would certainly raise an error if run.
This will call the C{may_run()} method on each task and return
false if any of them returns false.
"""
for task in self._tasks:
if not task.may_run(path):
return False
return True
def run(self, path):
"""Run all tasks with the given action.
@param path: The path of the expression run.
Running an event means running all of its tasks individually and in
order. An event should only ever be run if all of its tasks claim to
match the given action.
The result of this method will be the last result of a task
which isn't None, or None if they're all None.
"""
self._has_run = True
result = None
errors = []
for task in self._tasks:
if not errors or not task.may_run_user_code():
try:
task_result = task.run(path)
except AssertionError, e:
error = str(e)
if not error:
raise RuntimeError("Empty error message from %r" % task)
errors.append(error)
else:
# XXX That's actually a bit weird. What if a call() really
# returned None? This would improperly change the semantic
# of this process without any good reason. Test that with two
# call()s in sequence.
if task_result is not None:
result = task_result
if errors:
message = [str(self.path)]
if str(path) != message[0]:
message.append("- Run: %s" % path)
for error in errors:
lines = error.splitlines()
message.append("- " + lines.pop(0))
message.extend([" " + line for line in lines])
raise AssertionError(os.linesep.join(message))
return result
def satisfied(self):
"""Return true if all tasks are satisfied.
Being satisfied means that there are no unmet expectations.
"""
for task in self._tasks:
try:
task.verify()
except AssertionError:
return False
return True
def verify(self):
"""Run verify on all tasks.
The verify method is supposed to raise an AssertionError if the
task has unmet expectations, with a one-line explanation about
why this item is unmet. This method should be safe to be called
multiple times without side effects.
"""
errors = []
for task in self._tasks:
try:
task.verify()
except AssertionError, e:
error = str(e)
if not error:
raise RuntimeError("Empty error message from %r" % task)
errors.append(error)
if errors:
message = [str(self.path)]
for error in errors:
lines = error.splitlines()
message.append("- " + lines.pop(0))
message.extend([" " + line for line in lines])
raise AssertionError(os.linesep.join(message))
def replay(self):
"""Put all tasks in replay mode."""
self._has_run = False
for task in self._tasks:
task.replay()
def restore(self):
"""Restore the state of all tasks."""
for task in self._tasks:
task.restore()
class ReplayRestoreEvent(Event):
"""Helper event for tasks which need replay/restore but shouldn't match."""
def matches(self, path):
return False
class Task(object):
"""Element used to track one specific aspect on an event.
A task is responsible for adding any kind of logic to an event.
Examples of that are counting the number of times the event was
made, verifying parameters if any, and so on.
"""
def matches(self, path):
"""Return true if the task is supposed to be run for the given path.
"""
return True
def may_run(self, path):
"""Return false if running this task would certainly raise an error."""
return True
def may_run_user_code(self):
"""Return true if there's a chance this task may run custom code.
Whenever errors are detected, running user code should be avoided,
because the situation is already known to be incorrect, and any
errors in the user code are side effects rather than the cause.
"""
return False
def run(self, path):
"""Perform the task item, considering that the given action happened.
"""
def verify(self):
"""Raise AssertionError if expectations for this item are unmet.
The verify method is supposed to raise an AssertionError if the
task has unmet expectations, with a one-line explanation about
why this item is unmet. This method should be safe to be called
multiple times without side effects.
"""
def replay(self):
"""Put the task in replay mode.
Any expectations of the task should be reset.
"""
def restore(self):
"""Restore any environmental changes made by the task.
Verify should continue to work after this is called.
"""
# --------------------------------------------------------------------
# Task implementations.
class OnRestoreCaller(Task):
"""Call a given callback when restoring."""
def __init__(self, callback):
self._callback = callback
def restore(self):
self._callback()
class PathMatcher(Task):
"""Match the action path against a given path."""
def __init__(self, path):
self.path = path
def matches(self, path):
return self.path.matches(path)
def path_matcher_recorder(mocker, event):
event.add_task(PathMatcher(event.path))
Mocker.add_recorder(path_matcher_recorder)
class RunCounter(Task):
"""Task which verifies if the number of runs are within given boundaries.
"""
def __init__(self, min, max=False):
self.min = min
if max is None:
self.max = sys.maxint
elif max is False:
self.max = min
else:
self.max = max
self._runs = 0
def replay(self):
self._runs = 0
def may_run(self, path):
return self._runs < self.max
def run(self, path):
self._runs += 1
if self._runs > self.max:
self.verify()
def verify(self):
if not self.min <= self._runs <= self.max:
if self._runs < self.min:
raise AssertionError("Performed fewer times than expected.")
raise AssertionError("Performed more times than expected.")
class ImplicitRunCounter(RunCounter):
"""RunCounter inserted by default on any event.
This is a way to differentiate explicitly added counters and
implicit ones.
"""
def run_counter_recorder(mocker, event):
"""Any event may be repeated once, unless disabled by default."""
if event.path.root_mock.__mocker_count__:
# Rather than appending the task, we prepend it so that the
# issue is raised before any other side-effects happen.
event.prepend_task(ImplicitRunCounter(1))
Mocker.add_recorder(run_counter_recorder)
def run_counter_removal_recorder(mocker, event):
"""
Events created by getattr actions which lead to other events
may be repeated any number of times. For that, we remove implicit
run counters of any getattr actions leading to the current one.
"""
parent_path = event.path.parent_path
for event in mocker.get_events()[::-1]:
if (event.path is parent_path and
event.path.actions[-1].kind == "getattr"):
for task in event.get_tasks():
if type(task) is ImplicitRunCounter:
event.remove_task(task)
Mocker.add_recorder(run_counter_removal_recorder)
class MockReturner(Task):
"""Return a mock based on the action path."""
def __init__(self, mocker):
self.mocker = mocker
def run(self, path):
return Mock(self.mocker, path)
def mock_returner_recorder(mocker, event):
"""Events that lead to other events must return mock objects."""
parent_path = event.path.parent_path
for event in mocker.get_events():
if event.path is parent_path:
for task in event.get_tasks():
if isinstance(task, MockReturner):
break
else:
event.add_task(MockReturner(mocker))
break
Mocker.add_recorder(mock_returner_recorder)
class FunctionRunner(Task):
"""Task that runs a function everything it's run.
Arguments of the last action in the path are passed to the function,
and the function result is also returned.
"""
def __init__(self, func, with_root_object=False):
self._func = func
self._with_root_object = with_root_object
def may_run_user_code(self):
return True
def run(self, path):
action = path.actions[-1]
if self._with_root_object:
return self._func(path.root_object, *action.args, **action.kwargs)
else:
return self._func(*action.args, **action.kwargs)
class PathExecuter(Task):
"""Task that executes a path in the real object, and returns the result."""
def __init__(self, result_callback=None):
self._result_callback = result_callback
def get_result_callback(self):
return self._result_callback
def run(self, path):
result = path.execute(path.root_object)
if self._result_callback is not None:
self._result_callback(result)
return result
class Orderer(Task):
"""Task to establish an order relation between two events.
An orderer task will only match once all its dependencies have
been run.
"""
def __init__(self, path):
self.path = path
self._run = False
self._dependencies = []
def replay(self):
self._run = False
def has_run(self):
return self._run
def may_run(self, path):
for dependency in self._dependencies:
if not dependency.has_run():
return False
return True
def run(self, path):
for dependency in self._dependencies:
if not dependency.has_run():
raise AssertionError("Should be after: %s" % dependency.path)
self._run = True
def add_dependency(self, orderer):
self._dependencies.append(orderer)
def get_dependencies(self):
return self._dependencies
class SpecChecker(Task):
"""Task to check if arguments of the last action conform to a real method.
"""
def __init__(self, method):
self._method = method
self._unsupported = False
if method:
try:
self._args, self._varargs, self._varkwargs, self._defaults = \
inspect.getargspec(method)
except TypeError:
self._unsupported = True
else:
if self._defaults is None:
self._defaults = ()
if type(method) is type(self.run):
self._args = self._args[1:]
def get_method(self):
return self._method
def _raise(self, message):
spec = inspect.formatargspec(self._args, self._varargs,
self._varkwargs, self._defaults)
raise AssertionError("Specification is %s%s: %s" %
(self._method.__name__, spec, message))
def verify(self):
if not self._method:
raise AssertionError("Method not found in real specification")
def may_run(self, path):
try:
self.run(path)
except AssertionError:
return False
return True
def run(self, path):
if not self._method:
raise AssertionError("Method not found in real specification")
if self._unsupported:
return # Can't check it. Happens with builtin functions. :-(
action = path.actions[-1]
obtained_len = len(action.args)
obtained_kwargs = action.kwargs.copy()
nodefaults_len = len(self._args) - len(self._defaults)
for i, name in enumerate(self._args):
if i < obtained_len and name in action.kwargs:
self._raise("%r provided twice" % name)
if (i >= obtained_len and i < nodefaults_len and
name not in action.kwargs):
self._raise("%r not provided" % name)
obtained_kwargs.pop(name, None)
if obtained_len > len(self._args) and not self._varargs:
self._raise("too many args provided")
if obtained_kwargs and not self._varkwargs:
self._raise("unknown kwargs: %s" % ", ".join(obtained_kwargs))
def spec_checker_recorder(mocker, event):
spec = event.path.root_mock.__mocker_spec__
if spec:
actions = event.path.actions
if len(actions) == 1:
if actions[0].kind == "call":
method = getattr(spec, "__call__", None)
event.add_task(SpecChecker(method))
elif len(actions) == 2:
if actions[0].kind == "getattr" and actions[1].kind == "call":
method = getattr(spec, actions[0].args[0], None)
event.add_task(SpecChecker(method))
Mocker.add_recorder(spec_checker_recorder)
class ProxyReplacer(Task):
"""Task which installs and deinstalls proxy mocks.
This task will replace a real object by a mock in all dictionaries
found in the running interpreter via the garbage collecting system.
"""
def __init__(self, mock):
self.mock = mock
self.__mocker_replace__ = False
def replay(self):
global_replace(self.mock.__mocker_object__, self.mock)
def restore(self):
global_replace(self.mock, self.mock.__mocker_object__)
def global_replace(remove, install):
"""Replace object 'remove' with object 'install' on all dictionaries."""
for referrer in gc.get_referrers(remove):
if (type(referrer) is dict and
referrer.get("__mocker_replace__", True)):
for key, value in list(referrer.iteritems()):
if value is remove:
referrer[key] = install
class Undefined(object):
def __repr__(self):
return "Undefined"
Undefined = Undefined()
class Patcher(Task):
def __init__(self):
super(Patcher, self).__init__()
self._monitored = {} # {kind: {id(object): object}}
self._patched = {}
def is_monitoring(self, obj, kind):
monitored = self._monitored.get(kind)
if monitored:
if id(obj) in monitored:
return True
cls = type(obj)
if issubclass(cls, type):
cls = obj
bases = set([id(base) for base in cls.__mro__])
bases.intersection_update(monitored)
return bool(bases)
return False
def monitor(self, obj, kind):
if kind not in self._monitored:
self._monitored[kind] = {}
self._monitored[kind][id(obj)] = obj
def patch_attr(self, obj, attr, value):
original = obj.__dict__.get(attr, Undefined)
self._patched[id(obj), attr] = obj, attr, original
setattr(obj, attr, value)
def get_unpatched_attr(self, obj, attr):
cls = type(obj)
if issubclass(cls, type):
cls = obj
result = Undefined
for mro_cls in cls.__mro__:
key = (id(mro_cls), attr)
if key in self._patched:
result = self._patched[key][2]
if result is not Undefined:
break
elif attr in mro_cls.__dict__:
result = mro_cls.__dict__.get(attr, Undefined)
break
if isinstance(result, object) and hasattr(type(result), "__get__"):
if cls is obj:
obj = None
return result.__get__(obj, cls)
return result
def _get_kind_attr(self, kind):
if kind == "getattr":
return "__getattribute__"
return "__%s__" % kind
def replay(self):
for kind in self._monitored:
attr = self._get_kind_attr(kind)
seen = set()
for obj in self._monitored[kind].itervalues():
cls = type(obj)
if issubclass(cls, type):
cls = obj
if cls not in seen:
seen.add(cls)
unpatched = getattr(cls, attr, Undefined)
self.patch_attr(cls, attr,
PatchedMethod(kind, unpatched,
self.is_monitoring))
self.patch_attr(cls, "__mocker_execute__",
self.execute)
def restore(self):
for obj, attr, original in self._patched.itervalues():
if original is Undefined:
delattr(obj, attr)
else:
setattr(obj, attr, original)
self._patched.clear()
def execute(self, action, object):
attr = self._get_kind_attr(action.kind)
unpatched = self.get_unpatched_attr(object, attr)
try:
return unpatched(*action.args, **action.kwargs)
except AttributeError:
type, value, traceback = sys.exc_info()
if action.kind == "getattr":
# The normal behavior of Python is to try __getattribute__,
# and if it raises AttributeError, try __getattr__. We've
# tried the unpatched __getattribute__ above, and we'll now
# try __getattr__.
try:
__getattr__ = unpatched("__getattr__")
except AttributeError:
pass
else:
return __getattr__(*action.args, **action.kwargs)
raise type, value, traceback
class PatchedMethod(object):
def __init__(self, kind, unpatched, is_monitoring):
self._kind = kind
self._unpatched = unpatched
self._is_monitoring = is_monitoring
def __get__(self, obj, cls=None):
object = obj or cls
if not self._is_monitoring(object, self._kind):
return self._unpatched.__get__(obj, cls)
def method(*args, **kwargs):
if self._kind == "getattr" and args[0].startswith("__mocker_"):
return self._unpatched.__get__(obj, cls)(args[0])
mock = object.__mocker_mock__
return mock.__mocker_act__(self._kind, args, kwargs, object)
return method
def __call__(self, obj, *args, **kwargs):
# At least with __getattribute__, Python seems to use *both* the
# descriptor API and also call the class attribute directly. It
# looks like an interpreter bug, or at least an undocumented
# inconsistency. Coverage tests may show this uncovered, because
# it depends on the Python version.
return self.__get__(obj)(*args, **kwargs)
def patcher_recorder(mocker, event):
mock = event.path.root_mock
if mock.__mocker_patcher__ and len(event.path.actions) == 1:
patcher = mock.__mocker_patcher__
patcher.monitor(mock.__mocker_object__, event.path.actions[0].kind)
Mocker.add_recorder(patcher_recorder)
txzookeeper-0.9.8/txzookeeper/tests/test_queue.py 0000664 0001750 0001750 00000032760 11745322401 022620 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from zookeeper import NoNodeException
from twisted.internet.defer import (
inlineCallbacks, returnValue, DeferredList, Deferred, succeed, fail)
from txzookeeper import ZookeeperClient
from txzookeeper.client import NotConnectedException
from txzookeeper.queue import Queue, ReliableQueue, SerializedQueue, QueueItem
from txzookeeper.tests import ZookeeperTestCase, utils
class QueueTests(ZookeeperTestCase):
queue_factory = Queue
def setUp(self):
super(QueueTests, self).setUp()
self.clients = []
def tearDown(self):
cleanup = False
for client in self.clients:
if not cleanup and client.connected:
utils.deleteTree(handle=client.handle)
cleanup = True
if client.connected:
client.close()
super(QueueTests, self).tearDown()
def compare_data(self, data, item):
if isinstance(item, QueueItem):
self.assertEqual(data, item.data)
else:
self.assertEqual(data, item)
def consume_item(self, item):
if isinstance(item, QueueItem):
return item.delete(), item.data
return None, item
@inlineCallbacks
def open_client(self, credentials=None):
"""
Open a zookeeper client, optionally authenticating with the
credentials if given.
"""
client = ZookeeperClient("127.0.0.1:2181")
self.clients.append(client)
yield client.connect()
if credentials:
d = client.add_auth("digest", credentials)
# hack to keep auth fast
yield client.exists("/")
yield d
returnValue(client)
def test_path_property(self):
"""
The queue has a property that can be used to introspect its
path in read only manner.
"""
q = self.queue_factory("/moon", None)
self.assertEqual(q.path, "/moon")
def test_persistent_property(self):
"""
The queue has a property that can be used to introspect
whether or not the queue entries are persistent.
"""
q = self.queue_factory("/moon", None, persistent=True)
self.assertEqual(q.persistent, True)
@inlineCallbacks
def test_put_item(self):
"""
An item can be put on the queue, and is stored in a node in
queue's directory.
"""
client = yield self.open_client()
path = yield client.create("/queue-test")
queue = self.queue_factory(path, client)
item = "transform image bluemarble.jpg"
yield queue.put(item)
children = yield client.get_children(path)
self.assertEqual(len(children), 1)
data, stat = yield client.get("/".join((path, children[0])))
self.compare_data(data, item)
@inlineCallbacks
def test_qsize(self):
"""
The client implements a method which returns an unreliable
approximation of the number of items in the queue (mirrors api
of Queue.Queue), its unreliable only in that the value represents
a temporal snapshot of the value at the time it was requested,
not its current value.
"""
client = yield self.open_client()
path = yield client.create("/test-qsize")
queue = self.queue_factory(path, client)
yield queue.put("abc")
size = yield queue.qsize()
self.assertTrue(size, 1)
yield queue.put("bcd")
size = yield queue.qsize()
self.assertTrue(size, 2)
yield queue.get()
size = yield queue.qsize()
self.assertTrue(size, 1)
@inlineCallbacks
def test_invalid_put_item(self):
"""
The queue only accepts string items.
"""
client = yield self.open_client()
queue = self.queue_factory("/unused", client)
self.failUnlessFailure(queue.put(123), ValueError)
@inlineCallbacks
def test_get_with_invalid_queue(self):
"""
If the queue hasn't been created an unknown node exception is raised
on get.
"""
client = yield self.open_client()
queue = self.queue_factory("/unused", client)
yield self.failUnlessFailure(queue.put("abc"), NoNodeException)
@inlineCallbacks
def test_put_with_invalid_queue(self):
"""
If the queue hasn't been created an unknown node exception is raised
on put.
"""
client = yield self.open_client()
queue = self.queue_factory("/unused", client)
yield self.failUnlessFailure(queue.put("abc"), NoNodeException)
@inlineCallbacks
def test_unexpected_error_during_item_retrieval(self):
"""
If an unexpected error occurs when reserving an item, the error is
passed up to the get deferred's errback method.
"""
test_client = yield self.open_client()
path = yield test_client.create("/reliable-queue-test")
# setup the test scenario
mock_client = self.mocker.patch(test_client)
mock_client.get_children_and_watch(path)
watch = Deferred()
self.mocker.result((succeed(["entry-000000"]), watch))
item_path = "%s/%s" % (path, "entry-000000")
mock_client.get(item_path)
self.mocker.result(fail(SyntaxError("x")))
self.mocker.replay()
# odd behavior, this should return a failure, as above, but it returns
# None
d = self.queue_factory(path, mock_client).get()
assert d
self.failUnlessFailure(d, SyntaxError)
yield d
@inlineCallbacks
def test_get_and_put(self):
"""
Get can also be used on empty queues and returns a deferred that fires
whenever an item is has been retrieved from the queue.
"""
client = yield self.open_client()
path = yield client.create("/queue-wait-test")
data = "zebra moon"
queue = self.queue_factory(path, client)
d = queue.get()
@inlineCallbacks
def push_item():
queue = self.queue_factory(path, client)
yield queue.put(data)
from twisted.internet import reactor
reactor.callLater(0.1, push_item)
item = yield d
self.compare_data(data, item)
@inlineCallbacks
def test_interleaved_multiple_consumers_wait(self):
"""
Multiple consumers and a producer adding and removing items on the
the queue concurrently.
"""
test_client = yield self.open_client()
path = yield test_client.create("/multi-consumer-wait-test")
results = []
@inlineCallbacks
def producer(item_count):
from twisted.internet import reactor
client = yield self.open_client()
queue = self.queue_factory(path, client)
items = []
producer_done = Deferred()
def iteration(i):
if len(items) == (item_count - 1):
return producer_done.callback(None)
items.append(i)
queue.put(str(i))
for i in range(item_count):
reactor.callLater(i * 0.05, iteration, i)
yield producer_done
returnValue(items)
@inlineCallbacks
def consumer(item_count):
client = yield self.open_client()
queue = self.queue_factory(path, client)
for i in range(item_count):
try:
data = yield queue.get()
d, data = self.consume_item(data)
if d:
yield d
except NotConnectedException:
# when the test closes, we need to catch this
# as one of the producers will likely hang.
returnValue(len(results))
results.append((client.handle, data))
returnValue(len(results))
yield DeferredList(
[DeferredList([consumer(3), consumer(2)], fireOnOneCallback=1),
producer(6)])
# as soon as the producer and either consumer is complete than the test
# is done. Thus the only assertion we can make is the result is the
# size of at least the smallest consumer.
self.assertTrue(len(results) >= 2)
@inlineCallbacks
def test_staged_multiproducer_multiconsumer(self):
"""
A real world scenario test, A set of producers filling a queue with
items, and then a set of concurrent consumers pulling from the queue
till its empty. The consumers use a non blocking get (defer raises
exception on empty).
"""
test_client = yield self.open_client()
path = yield test_client.create("/multi-prod-cons")
consume_results = []
produce_results = []
@inlineCallbacks
def producer(start, offset):
client = yield self.open_client()
q = self.queue_factory(path, client)
for i in range(start, start + offset):
yield q.put(str(i))
produce_results.append(str(i))
@inlineCallbacks
def consumer(max):
client = yield self.open_client()
q = self.queue_factory(path, client)
attempts = range(max)
for el in attempts:
value = yield q.get()
d, value = self.consume_item(value)
if d:
yield d
consume_results.append(value)
returnValue(True)
# two producers 20 items total
yield DeferredList(
[producer(0, 10), producer(10, 10)])
children = yield test_client.get_children(path)
self.assertEqual(len(children), 20)
yield DeferredList(
[consumer(8), consumer(8), consumer(4)])
err = set(produce_results) - set(consume_results)
self.assertFalse(err)
self.assertEqual(len(consume_results), len(produce_results))
class ReliableQueueTests(QueueTests):
queue_factory = ReliableQueue
@inlineCallbacks
def test_unprocessed_item_reappears(self):
"""
If a queue consumer exits before processing an item, then
the item will become visible to other queue consumers.
"""
test_client = yield self.open_client()
path = yield test_client.create("/reliable-queue-test")
data = "rabbit stew"
queue = self.queue_factory(path, test_client)
yield queue.put(data)
test_client2 = yield self.open_client()
queue2 = self.queue_factory(path, test_client2)
item = yield queue2.get()
self.compare_data(data, item)
d = queue.get()
yield test_client2.close()
item = yield d
self.compare_data(data, item)
@inlineCallbacks
def test_processed_item_removed(self):
"""
If a client processes an item, than that item is removed from the queue
permanently.
"""
test_client = yield self.open_client()
path = yield test_client.create("/reliable-queue-test")
data = "rabbit stew"
queue = self.queue_factory(path, test_client)
yield queue.put(data)
item = yield queue.get()
self.compare_data(data, item)
yield item.delete()
yield test_client.close()
test_client2 = yield self.open_client()
children = yield test_client2.get_children(path)
children = [c for c in children if c.startswith(queue.prefix)]
self.assertFalse(bool(children))
class SerializedQueueTests(ReliableQueueTests):
queue_factory = SerializedQueue
@inlineCallbacks
def test_serialized_behavior(self):
"""
The serialized queue behavior is such that even with multiple
consumers, items are processed in order.
"""
test_client = yield self.open_client()
path = yield test_client.create("/serialized-queue-test")
queue = self.queue_factory(path, test_client, persistent=True)
yield queue.put("a")
yield queue.put("b")
test_client2 = yield self.open_client()
queue2 = self.queue_factory(path, test_client2, persistent=True)
d = queue2.get()
def on_get_item_sleep_and_close(item):
"""Close the connection after we have the item."""
from twisted.internet import reactor
reactor.callLater(0.1, test_client2.close)
return item
d.addCallback(on_get_item_sleep_and_close)
# fetch the item from queue2
item1 = yield d
# fetch the item from queue1, this will not get "b", because client2 is
# still processing "a". When client2 closes its connection, client1
# will get item "a"
item2 = yield queue.get()
self.compare_data("a", item2)
self.assertEqual(item1.data, item2.data)
txzookeeper-0.9.8/txzookeeper/tests/test_node.py 0000664 0001750 0001750 00000027670 12144707107 022431 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import base64
import hashlib
import zookeeper
from twisted.internet.defer import inlineCallbacks
from twisted.python.failure import Failure
from txzookeeper.node import ZNode
from txzookeeper.node import NodeEvent
from txzookeeper.tests import TestCase
from txzookeeper.tests.utils import deleteTree
from txzookeeper import ZookeeperClient
class NodeTest(TestCase):
def setUp(self):
super(NodeTest, self).setUp()
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_ERROR)
self.client = ZookeeperClient("127.0.0.1:2181", 2000)
d = self.client.connect()
self.client2 = None
def create_zoo(client):
client.create("/zoo")
d.addCallback(create_zoo)
return d
def tearDown(self):
super(NodeTest, self).tearDown()
deleteTree(handle=self.client.handle)
if self.client.connected:
self.client.close()
if self.client2 and self.client2.connected:
self.client2.close()
zookeeper.set_debug_level(zookeeper.LOG_LEVEL_DEBUG)
def _make_digest_identity(self, credentials):
user, password = credentials.split(":")
digest = hashlib.new("sha1", credentials).digest()
return "%s:%s" % (user, base64.b64encode(digest))
def test_node_name_and_path(self):
"""
Each node has name and path.
"""
node = ZNode("/zoo/rabbit", self.client)
self.assertEqual(node.name, "rabbit")
self.assertEqual(node.path, "/zoo/rabbit")
def test_node_event_repr(self):
"""
Node events have a human-readable representation.
"""
node = ZNode("/zoo", self.client)
event = NodeEvent(4, None, node)
self.assertEqual(repr(event), "")
@inlineCallbacks
def test_node_exists_nonexistant(self):
"""
A node knows whether it exists or not.
"""
node = ZNode("/zoo/rabbit", self.client)
exists = yield node.exists()
self.assertFalse(exists)
@inlineCallbacks
def test_node_set_data_on_nonexistant(self):
"""
Setting data on a non existant node raises a no node exception.
"""
node = ZNode("/zoo/rabbit", self.client)
d = node.set_data("big furry ears")
self.failUnlessFailure(d, zookeeper.NoNodeException)
yield d
@inlineCallbacks
def test_node_create_set_data(self):
"""
A node can be created and have its data set.
"""
node = ZNode("/zoo/rabbit", self.client)
data = "big furry ears"
yield node.create(data)
exists = yield self.client.exists("/zoo/rabbit")
self.assertTrue(exists)
node_data = yield node.get_data()
self.assertEqual(data, node_data)
data = data*2
yield node.set_data(data)
node_data = yield node.get_data()
self.assertEqual(data, node_data)
@inlineCallbacks
def test_node_get_data(self):
"""
Data can be fetched from a node.
"""
yield self.client.create("/zoo/giraffe", "mouse")
data = yield ZNode("/zoo/giraffe", self.client).get_data()
self.assertEqual(data, "mouse")
@inlineCallbacks
def test_node_get_data_nonexistant(self):
"""
Attempting to fetch data from a nonexistant node returns
a non existant error.
"""
d = ZNode("/zoo/giraffe", self.client).get_data()
self.failUnlessFailure(d, zookeeper.NoNodeException)
yield d
@inlineCallbacks
def test_node_get_acl(self):
"""
The ACL for a node can be retrieved.
"""
yield self.client.create("/zoo/giraffe")
acl = yield ZNode("/zoo/giraffe", self.client).get_acl()
self.assertEqual(len(acl), 1)
self.assertEqual(acl[0]['scheme'], 'world')
def test_node_get_acl_nonexistant(self):
"""
The fetching the ACL for a non-existant node results in an error.
"""
node = ZNode("/zoo/giraffe", self.client)
def assert_failed(failed):
if not isinstance(failed, Failure):
self.fail("Should have failed")
self.assertTrue(
isinstance(failed.value, zookeeper.NoNodeException))
d = node.get_acl()
d.addBoth(assert_failed)
return d
@inlineCallbacks
def test_node_set_acl(self):
"""
The ACL for a node can be modified.
"""
path = yield self.client.create("/zoo/giraffe")
credentials = "zebra:moon"
acl = [{"id": self._make_digest_identity(credentials),
"scheme": "digest",
"perms":zookeeper.PERM_ALL}]
node = ZNode(path, self.client)
# little hack around slow auth issue 770 zookeeper
d = self.client.add_auth("digest", credentials)
yield node.set_acl(acl)
yield d
node_acl, stat = yield self.client.get_acl(path)
self.assertEqual(node_acl, acl)
@inlineCallbacks
def test_node_set_data_update_with_cached_exists(self):
"""
Data can be set on an existing node, updating it
in place.
"""
node = ZNode("/zoo/monkey", self.client)
yield self.client.create("/zoo/monkey", "stripes")
exists = yield node.exists()
self.assertTrue(exists)
yield node.set_data("banana")
data, stat = yield self.client.get("/zoo/monkey")
self.assertEqual(data, "banana")
@inlineCallbacks
def test_node_set_data_update_with_invalid_cached_exists(self):
"""
If a node is deleted, attempting to set data on it
raises a no node exception.
"""
node = ZNode("/zoo/monkey", self.client)
yield self.client.create("/zoo/monkey", "stripes")
exists = yield node.exists()
self.assertTrue(exists)
yield self.client.delete("/zoo/monkey")
d = node.set_data("banana")
self.failUnlessFailure(d, zookeeper.NoNodeException)
yield d
@inlineCallbacks
def test_node_set_data_update_with_exists(self):
"""
Data can be set on an existing node, updating it
in place.
"""
node = ZNode("/zoo/monkey", self.client)
yield self.client.create("/zoo/monkey", "stripes")
yield node.set_data("banana")
data, stat = yield self.client.get("/zoo/monkey")
self.assertEqual(data, "banana")
@inlineCallbacks
def test_node_exists_with_watch_nonexistant(self):
"""
The node's existance can be checked with the exist_watch api
a deferred will be returned and any node level events,
created, deleted, modified invoke the callback. You can
get these create event callbacks for non existant nodes.
"""
node = ZNode("/zoo/elephant", self.client)
exists, watch = yield node.exists_and_watch()
self.assertFalse((yield exists))
yield self.client.create("/zoo/elephant")
event = yield watch
self.assertEqual(event.type, zookeeper.CREATED_EVENT)
self.assertEqual(event.path, node.path)
@inlineCallbacks
def test_node_get_data_with_watch_on_update(self):
"""
Subscribing to a node will get node update events.
"""
yield self.client.create("/zoo/elephant")
node = ZNode("/zoo/elephant", self.client)
data, watch = yield node.get_data_and_watch()
yield self.client.set("/zoo/elephant")
event = yield watch
self.assertEqual(event.type, zookeeper.CHANGED_EVENT)
self.assertEqual(event.path, "/zoo/elephant")
@inlineCallbacks
def test_node_get_data_with_watch_on_delete(self):
"""
Subscribing to a node will get node deletion events.
"""
yield self.client.create("/zoo/elephant")
node = ZNode("/zoo/elephant", self.client)
data, watch = yield node.get_data_and_watch()
yield self.client.delete("/zoo/elephant")
event = yield watch
self.assertEqual(event.type, zookeeper.DELETED_EVENT)
self.assertEqual(event.path, "/zoo/elephant")
@inlineCallbacks
def test_node_children(self):
"""
A node's children can be introspected.
"""
node = ZNode("/zoo", self.client)
node_path_a = yield self.client.create("/zoo/lion")
node_path_b = yield self.client.create("/zoo/tiger")
children = yield node.get_children()
children.sort()
self.assertEqual(children[0].path, node_path_a)
self.assertEqual(children[1].path, node_path_b)
@inlineCallbacks
def test_node_children_by_prefix(self):
"""
A node's children can be introspected optionally with a prefix.
"""
node = ZNode("/zoo", self.client)
node_path_a = yield self.client.create("/zoo/lion")
yield self.client.create("/zoo/tiger")
children = yield node.get_children("lion")
children.sort()
self.assertEqual(children[0].path, node_path_a)
self.assertEqual(len(children), 1)
@inlineCallbacks
def test_node_get_children_with_watch_create(self):
"""
A node's children can explicitly be watched to given existance
events for node creation and destruction.
"""
node = ZNode("/zoo", self.client)
children, watch = yield node.get_children_and_watch()
yield self.client.create("/zoo/lion")
event = yield watch
self.assertEqual(event.path, "/zoo")
self.assertEqual(event.type, zookeeper.CHILD_EVENT)
self.assertEqual(event.type_name, "child")
@inlineCallbacks
def test_node_get_children_with_watch_delete(self):
"""
A node's children can explicitly be watched to given existance
events for node creation and destruction.
"""
node = ZNode("/zoo", self.client)
yield self.client.create("/zoo/lion")
children, watch = yield node.get_children_and_watch()
yield self.client.delete("/zoo/lion")
event = yield watch
self.assertEqual(event.path, "/zoo")
self.assertEqual(event.type, zookeeper.CHILD_EVENT)
@inlineCallbacks
def test_bad_version_error(self):
"""
The node captures the node version on any read operations, which
it utilizes for write operations. On a concurrent modification error
the node return a bad version error, this also clears the cached
state so subsequent modifications will be against the latest version,
unless the cache is seeded again by a read operation.
"""
node = ZNode("/zoo/lion", self.client)
self.client2 = ZookeeperClient("127.0.0.1:2181")
yield self.client2.connect()
yield self.client.create("/zoo/lion", "mouse")
yield node.get_data()
yield self.client2.set("/zoo/lion", "den2")
data = yield self.client.exists("/zoo/lion")
self.assertEqual(data['version'], 1)
d = node.set_data("zebra")
self.failUnlessFailure(d, zookeeper.BadVersionException)
yield d
# after failure the cache is deleted, and a set proceeds
yield node.set_data("zebra")
data = yield node.get_data()
self.assertEqual(data, "zebra")
txzookeeper-0.9.8/txzookeeper/tests/test_conn_failure.py 0000664 0001750 0001750 00000015401 12144707107 024135 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import zookeeper
from twisted.internet import reactor
from twisted.internet.defer import inlineCallbacks
from txzookeeper import ZookeeperClient
from txzookeeper.tests import ZookeeperTestCase, utils
from txzookeeper.tests.proxy import ProxyFactory
class WatchDeliveryConnectionFailedTest(ZookeeperTestCase):
"""Watches are still sent on reconnect.
"""
def setUp(self):
super(WatchDeliveryConnectionFailedTest, self).setUp()
self.proxy = ProxyFactory("127.0.0.1", 2181)
self.proxy_port = reactor.listenTCP(0, self.proxy)
host = self.proxy_port.getHost()
self.proxied_client = ZookeeperClient(
"%s:%s" % (host.host, host.port))
self.direct_client = ZookeeperClient("127.0.0.1:2181", 3000)
self.session_events = []
def session_event_collector(conn, event):
self.session_events.append(event)
self.proxied_client.set_session_callback(session_event_collector)
return self.direct_client.connect()
@inlineCallbacks
def tearDown(self):
zookeeper.set_debug_level(0)
if self.proxied_client.connected:
yield self.proxied_client.close()
if not self.direct_client.connected:
yield self.direct_client.connect()
utils.deleteTree(handle=self.direct_client.handle)
yield self.direct_client.close()
self.proxy.lose_connection()
yield self.proxy_port.stopListening()
def verify_events(self, events, expected):
"""Verify the state of the session events encountered.
"""
for value, state in zip([e.state_name for e in events], expected):
self.assertEqual(value, state)
@inlineCallbacks
def test_child_watch_fires_upon_reconnect(self):
yield self.proxied_client.connect()
# Setup tree
cpath = "/test-tree"
yield self.direct_client.create(cpath)
# Setup watch
child_d, watch_d = self.proxied_client.get_children_and_watch(cpath)
self.assertEqual((yield child_d), [])
# Kill the connection and fire the watch
self.proxy.lose_connection()
yield self.direct_client.create(
cpath + "/abc", flags=zookeeper.SEQUENCE)
# We should still get the child event.
yield watch_d
# We get two pairs of (connecting, connected) for the conn and watch
self.assertEqual(len(self.session_events), 4)
self.verify_events(
self.session_events,
("connecting", "connecting", "connected", "connected"))
@inlineCallbacks
def test_exists_watch_fires_upon_reconnect(self):
yield self.proxied_client.connect()
cpath = "/test"
# Setup watch
exists_d, watch_d = self.proxied_client.exists_and_watch(cpath)
self.assertEqual((yield exists_d), None)
# Kill the connection and fire the watch
self.proxy.lose_connection()
yield self.direct_client.create(cpath)
# We should still get the exists event.
yield watch_d
# We get two pairs of (connecting, connected) for the conn and watch
self.assertEqual(len(self.session_events), 4)
self.verify_events(
self.session_events,
("connecting", "connecting", "connected", "connected"))
@inlineCallbacks
def test_get_watch_fires_upon_reconnect(self):
yield self.proxied_client.connect()
# Setup tree
cpath = "/test"
yield self.direct_client.create(cpath, "abc")
# Setup watch
get_d, watch_d = self.proxied_client.get_and_watch(cpath)
content, stat = yield get_d
self.assertEqual(content, "abc")
# Kill the connection and fire the watch
self.proxy.lose_connection()
yield self.direct_client.set(cpath, "xyz")
# We should still get the exists event.
yield watch_d
# We also two pairs of (connecting, connected) for the conn and watch
self.assertEqual(len(self.session_events), 4)
self.verify_events(
self.session_events,
("connecting", "connecting", "connected", "connected"))
@inlineCallbacks
def test_watch_delivery_failure_resends(self):
"""Simulate a network failure for the watch delivery
The zk server effectively sends the watch delivery to the client,
but the client never recieves it.
"""
yield self.proxied_client.connect()
cpath = "/test"
# Setup watch
exists_d, watch_d = self.proxied_client.exists_and_watch(cpath)
self.assertEqual((yield exists_d), None)
# Pause the connection fire the watch, and blackhole the data.
self.proxy.set_blocked(True)
yield self.direct_client.create(cpath)
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# We should still get the exists event.
yield watch_d
@inlineCallbacks
def xtest_binding_bug_session_exception(self):
"""This test triggers an exception in the python-zookeeper binding.
File "txzookeeper/client.py", line 491, in create
self.handle, path, data, acls, flags, callback)
exceptions.SystemError: error return without exception set
"""
yield self.proxied_client.connect()
data_d, watch_d = yield self.proxied_client.exists_and_watch("/")
self.assertTrue((yield data_d))
self.proxy.set_blocked(True)
# Wait for session expiration, on a single server options are limited
yield self.sleep(15)
# Unblock the proxy for next connect, and then drop the connection.
self.proxy.set_blocked(False)
self.proxy.lose_connection()
# Wait for a reconnect
yield self.assertFailure(watch_d, zookeeper.SessionExpiredException)
# Leads to bindings bug failure
yield self.assertFailure(
self.proxied_client.get("/a"),
zookeeper.SessionExpiredException)
self.assertEqual(self.session_events[-1].state_name, "expired")
txzookeeper-0.9.8/txzookeeper/tests/test_utils.py 0000664 0001750 0001750 00000014023 11745322401 022624 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import zookeeper
from twisted.internet.defer import inlineCallbacks, fail, succeed
from txzookeeper import ZookeeperClient
from txzookeeper.utils import retry_change
from txzookeeper.tests.mocker import MATCH
from txzookeeper.tests import ZookeeperTestCase, utils
MATCH_STAT = MATCH(lambda x: isinstance(x, dict))
class RetryChangeTest(ZookeeperTestCase):
def update_function_increment(self, content, stat):
if not content:
return str(0)
return str(int(content) + 1)
def setUp(self):
super(RetryChangeTest, self).setUp()
self.client = ZookeeperClient("127.0.0.1:2181")
return self.client.connect()
def tearDown(self):
utils.deleteTree("/", self.client.handle)
self.client.close()
@inlineCallbacks
def test_node_create(self):
"""
retry_change will create a node if one does not exist.
"""
#use a mock to ensure the change function is only invoked once
func = self.mocker.mock()
func(None, None)
self.mocker.result("hello")
self.mocker.replay()
yield retry_change(
self.client, "/magic-beans", func)
content, stat = yield self.client.get("/magic-beans")
self.assertEqual(content, "hello")
self.assertEqual(stat["version"], 0)
@inlineCallbacks
def test_node_update(self):
"""
retry_change will update an existing node.
"""
#use a mock to ensure the change function is only invoked once
func = self.mocker.mock()
func("", MATCH_STAT)
self.mocker.result("hello")
self.mocker.replay()
yield self.client.create("/magic-beans")
yield retry_change(
self.client, "/magic-beans", func)
content, stat = yield self.client.get("/magic-beans")
self.assertEqual(content, "hello")
self.assertEqual(stat["version"], 1)
def test_error_in_change_function_propogates(self):
"""
an error in the change function propogates to the caller.
"""
def error_function(content, stat):
raise SyntaxError()
d = retry_change(self.client, "/magic-beans", error_function)
self.failUnlessFailure(d, SyntaxError)
return d
@inlineCallbacks
def test_concurrent_update_bad_version(self):
"""
If the node is updated after the retry function has read
the node but before the content is set, the retry function
will perform another read/change_func/set cycle.
"""
yield self.client.create("/animals")
content, stat = yield self.client.get("/animals")
yield self.client.set("/animals", "5")
real_get = self.client.get
p_client = self.mocker.proxy(self.client)
p_client.get("/animals")
self.mocker.result(succeed((content, stat)))
p_client.get("/animals")
self.mocker.call(real_get)
self.mocker.replay()
yield retry_change(
p_client, "/animals", self.update_function_increment)
content, stat = yield real_get("/animals")
self.assertEqual(content, "6")
self.assertEqual(stat["version"], 2)
@inlineCallbacks
def test_create_node_exists(self):
"""
If the node is created after the retry function has determined
the node doesn't exist but before the node is created by the
retry function. the retry function will perform another
read/change_func/set cycle.
"""
yield self.client.create("/animals", "5")
real_get = self.client.get
p_client = self.mocker.patch(self.client)
p_client.get("/animals")
self.mocker.result(fail(zookeeper.NoNodeException()))
p_client.get("/animals")
self.mocker.call(real_get)
self.mocker.replay()
yield retry_change(
p_client, "/animals", self.update_function_increment)
content, stat = yield real_get("/animals")
self.assertEqual(content, "6")
self.assertEqual(stat["version"], 1)
def test_set_node_does_not_exist(self):
"""
if the retry function goes to update a node which has been
deleted since it was read, it will cycle through to another
read/change_func set cycle.
"""
real_get = self.client.get
p_client = self.mocker.patch(self.client)
p_client.get("/animals")
self.mocker.result("5", {"version": 1})
p_client.get("/animals")
self.mocker.call(real_get)
yield retry_change(
p_client, "/animals", self.update_function_increment)
content, stat = yield real_get("/animals")
self.assertEqual(content, "0")
self.assertEqual(stat["version"], 0)
def test_identical_content_noop(self):
"""
If the change function generates identical content to
the existing node, the retry change function exits without
modifying the node.
"""
self.client.create("/animals", "hello")
def update(content, stat):
return content
yield retry_change(self.client, "/animals", update)
content, stat = self.client.get("/animals")
self.assertEqual(content, "hello")
self.assertEqual(stat["version"], 0)
txzookeeper-0.9.8/txzookeeper/tests/test_client.py 0000664 0001750 0001750 00000116510 12144707107 022752 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011, 2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import base64
import hashlib
from twisted.internet.defer import Deferred, maybeDeferred
from twisted.internet.base import DelayedCall
from twisted.python.failure import Failure
import zookeeper
from mocker import ANY, MATCH, ARGS
from txzookeeper.tests import ZookeeperTestCase, utils
from txzookeeper.client import (
ZookeeperClient, ZOO_OPEN_ACL_UNSAFE, ConnectionTimeoutException,
ConnectionException, NotConnectedException, ClientEvent)
PUBLIC_ACL = ZOO_OPEN_ACL_UNSAFE
def match_deferred(arg):
return isinstance(arg, Deferred)
DEFERRED_MATCH = MATCH(match_deferred)
class ClientTests(ZookeeperTestCase):
def setUp(self):
super(ClientTests, self).setUp()
self.client = ZookeeperClient("127.0.0.1:2181", 3000)
self.client2 = None
def tearDown(self):
if self.client.connected:
utils.deleteTree(handle=self.client.handle)
self.client.close()
if self.client2 and self.client2.connected:
self.client2.close()
super(ClientTests, self).tearDown()
def test_wb_connect_after_timeout(self):
"""
Test an odd error scenario. If the zookeeper client succeeds in
connecting after a timeout, the connection should be closed, as
the connect deferred has already fired.
"""
mock_client = self.mocker.patch(self.client)
mock_client.close()
def close_state():
# Ensure the client state variable is correct after the close call.
self.client.connected = False
self.mocker.call(close_state)
self.mocker.replay()
task = DelayedCall(1, lambda: 1, None, None, None, None)
task.called = True
d = Deferred()
d.errback(ConnectionTimeoutException())
self.client._cb_connected(
task, d, None, zookeeper.CONNECTED_STATE, "/")
self.failUnlessFailure(d, ConnectionTimeoutException)
return d
def test_wb_reconnect_after_timeout_and_close(self):
"""
Another odd error scenario, if a client instance has has
connect and closed methods invoked in succession multiple
times, and a previous callback connect timeouts, the callback
of a previous connect can be invoked by a subsequent connect,
with a CONNECTING_STATE. Verify this does not attempt to
invoke the connect deferred again.
"""
d = Deferred()
d.callback(True)
task = DelayedCall(1, lambda: 1, None, None, None, None)
task.called = True
self.assertEqual(
self.client._cb_connected(
task, d, None, zookeeper.CONNECTING_STATE, ""),
None)
def test_connect(self):
"""
The client can connect to a zookeeper instance.
"""
d = self.client.connect()
def check_connected(client):
self.assertEquals(client.connected, True)
self.assertEquals(client.state, zookeeper.CONNECTED_STATE)
d.addCallback(check_connected)
return d
def test_close(self):
"""
Test that the connection is closed, also for the first
connection when the zookeeper handle is 0.
"""
def _fake_init(*_):
return 0
mock_init = self.mocker.replace("zookeeper.init")
mock_init(ARGS)
self.mocker.call(_fake_init)
def _fake_close(handle):
return zookeeper.OK
mock_close = self.mocker.replace("zookeeper.close")
mock_close(0)
self.mocker.call(_fake_close)
self.mocker.replay()
# Avoid unclean reactor by letting the callLater go through,
# but we do not care about the timeout.
def _silence_timeout(failure):
failure.trap(ConnectionTimeoutException)
self.client.connect(timeout=0).addErrback(_silence_timeout)
d = maybeDeferred(self.client.close)
def _verify(result):
self.mocker.verify()
d.addCallback(_verify)
return d
def test_client_event_repr(self):
event = ClientEvent(zookeeper.SESSION_EVENT,
zookeeper.EXPIRED_SESSION_STATE, '', 0)
self.assertEqual(
repr(event),
"")
def test_client_event_attributes(self):
event = ClientEvent(4, 'state', 'path', 0)
self.assertEqual(event.type, 4)
self.assertEqual(event.connection_state, 'state')
self.assertEqual(event.path, 'path')
self.assertEqual(event.handle, 0)
self.assertEqual(event, (4, 'state', 'path', 0))
def test_client_use_while_disconnected_returns_failure(self):
return self.assertFailure(
self.client.exists("/"), NotConnectedException)
def test_create_ephemeral_node_and_close_connection(self):
"""
The client can create transient nodes that are destroyed when the
client is closed and the session is destroyed on the zookeeper servers.
"""
d = self.client.connect()
def test_create_ephemeral_node(client):
d = self.client.create(
"/foobar-transient", "rabbit", flags=zookeeper.EPHEMERAL)
return d
def check_node_path(path):
self.assertEqual(path, "/foobar-transient")
return path
def close_connection(path):
return self.client.close()
def new_connection(close_result):
self.client2 = new_client = ZookeeperClient("127.0.0.1:2181")
return new_client.connect()
def check_node_doesnt_exist(connected):
self.assertRaises(
zookeeper.NoNodeException,
zookeeper.get,
connected.handle,
"/foobar-transient")
self.client2.close()
d.addCallback(test_create_ephemeral_node)
d.addCallback(check_node_path)
d.addCallback(close_connection)
d.addCallback(new_connection)
d.addCallback(check_node_doesnt_exist)
return d
def test_create_node(self):
"""
We can create a node in zookeeper, with a given path
"""
d = self.client.connect()
def create_ephemeral_node(connected):
d = self.client.create(
"/foobar", "rabbit", flags=zookeeper.EPHEMERAL)
return d
def verify_node_path_and_content(path):
self.assertEqual(path, "/foobar")
self.assertNotEqual(
zookeeper.exists(self.client.handle, path), None)
data, stat = zookeeper.get(self.client.handle, path)
self.assertEqual(data, "rabbit")
d.addCallback(create_ephemeral_node)
d.addCallback(verify_node_path_and_content)
return d
def test_create_persistent_node_and_close(self):
"""
The client creates persistent nodes by default that exist independently
of the client session.
"""
d = self.client.connect()
def test_create_ephemeral_node(client):
d = self.client.create(
"/foobar-persistent", "rabbit")
return d
def check_node_path(path):
self.assertEqual(path, "/foobar-persistent")
self.assertNotEqual(
zookeeper.exists(self.client.handle, path), None)
return path
def close_connection(path):
self.client.close()
self.client2 = new_client = ZookeeperClient("127.0.0.1:2181")
return new_client.connect()
def check_node_exists(client):
data, stat = zookeeper.get(client.handle, "/foobar-persistent")
self.assertEqual(data, "rabbit")
d.addCallback(test_create_ephemeral_node)
d.addCallback(check_node_path)
d.addCallback(close_connection)
d.addCallback(check_node_exists)
return d
def test_get(self):
"""
The client can retrieve a node's data via its get method.
"""
d = self.client.connect()
def create_node(client):
d = self.client.create(
"/foobar-transient", "rabbit", flags=zookeeper.EPHEMERAL)
return d
def get_contents(path):
return self.client.get(path)
def verify_contents((data, stat)):
self.assertEqual(data, "rabbit")
d.addCallback(create_node)
d.addCallback(get_contents)
d.addCallback(verify_contents)
return d
def test_get_with_error(self):
"""
On get error the deferred's errback is raised.
"""
d = self.client.connect()
def get_contents(client):
return client.get("/foobar-transient")
def verify_failure(failure):
self.assertTrue(
isinstance(failure.value, zookeeper.NoNodeException))
def assert_failure(extra):
self.fail("get should have failed")
d.addCallback(get_contents)
d.addCallback(verify_failure)
d.addErrback(verify_failure)
return d
def test_get_with_watcher(self):
"""
The client can specify a callable watcher when invoking get. The
watcher will be called back when the client path is modified in
another session.
"""
d = self.client.connect()
watch_deferred = Deferred()
def create_node(client):
return self.client.create("/foobar-watched", "rabbit")
def get_node(path):
data, watch = self.client.get_and_watch(path)
watch.chainDeferred(watch_deferred)
return data
def new_connection(data):
self.client2 = ZookeeperClient("127.0.0.1:2181")
return self.client2.connect()
def trigger_watch(client):
zookeeper.set(self.client2.handle, "/foobar-watched", "abc")
return watch_deferred
def verify_watch(event):
self.assertEqual(event.path, "/foobar-watched")
self.assertEqual(event.type, zookeeper.CHANGED_EVENT)
d.addCallback(create_node)
d.addCallback(get_node)
d.addCallback(new_connection)
d.addCallback(trigger_watch)
d.addCallback(verify_watch)
return d
def test_get_with_watcher_and_delete(self):
"""
The client can specify a callable watcher when invoking get. The
watcher will be called back when the client path is modified in
another session.
"""
d = self.client.connect()
def create_node(client):
return self.client.create("/foobar-watched", "rabbit")
def get_node(path):
data, watch = self.client.get_and_watch(path)
return data.addCallback(lambda x: (watch,))
def new_connection((watch,)):
self.client2 = ZookeeperClient("127.0.0.1:2181")
return self.client2.connect().addCallback(
lambda x, y=None, z=None: (x, watch))
def trigger_watch((client, watch)):
zookeeper.delete(self.client2.handle, "/foobar-watched")
self.client2.close()
return watch
def verify_watch(event):
self.assertEqual(event.path, "/foobar-watched")
self.assertEqual(event.type, zookeeper.DELETED_EVENT)
d.addCallback(create_node)
d.addCallback(get_node)
d.addCallback(new_connection)
d.addCallback(trigger_watch)
d.addCallback(verify_watch)
return d
def test_delete(self):
"""
The client can delete a node via its delete method.
"""
d = self.client.connect()
def create_node(client):
return self.client.create(
"/foobar-transient", "rabbit", flags=zookeeper.EPHEMERAL)
def verify_exists(path):
self.assertNotEqual(
zookeeper.exists(self.client.handle, path), None)
return path
def delete_node(path):
return self.client.delete(path)
def verify_not_exists(*args):
self.assertEqual(
zookeeper.exists(self.client.handle, "/foobar-transient"),
None)
d.addCallback(create_node)
d.addCallback(verify_exists)
d.addCallback(delete_node)
d.addCallback(verify_not_exists)
return d
def test_exists_with_existing(self):
"""
The exists method returns node stat information for an existing node.
"""
d = self.client.connect()
def create_node(client):
return self.client.create(
"/foobar-transient", "rabbit", flags=zookeeper.EPHEMERAL)
def check_exists(path):
return self.client.exists(path)
def verify_exists(node_stat):
self.assertEqual(node_stat["dataLength"], 6)
self.assertEqual(node_stat["version"], 0)
d.addCallback(create_node)
d.addCallback(check_exists)
d.addCallback(verify_exists)
return d
def test_exists_with_error(self):
"""
On error exists invokes the errback with the exception.
"""
d = self.client.connect()
def inject_error(result_code, d, extra_codes=None, path=None):
error = SyntaxError()
d.errback(error)
return error
def check_exists(client):
mock_client = self.mocker.patch(client)
mock_client._check_result(
ANY, DEFERRED_MATCH, extra_codes=(zookeeper.NONODE,),
path="/zebra-moon")
self.mocker.call(inject_error)
self.mocker.replay()
return client.exists("/zebra-moon")
def verify_failure(failure):
self.assertTrue(isinstance(failure.value, SyntaxError))
d.addCallback(check_exists)
d.addErrback(verify_failure)
return d
def test_exists_with_nonexistant(self):
"""
The exists method returns None when the value node doesn't exist.
"""
d = self.client.connect()
def check_exists(client):
return self.client.exists("/abcdefg")
def verify_exists(node_stat):
self.assertEqual(node_stat, None)
d.addCallback(check_exists)
d.addCallback(verify_exists)
return d
def test_exist_watch_with_node_change(self):
"""
Setting an exist watches on existing node will also respond to
node changes.
"""
d = self.client.connect()
def create_node(client):
return client.create("/rome")
def check_exists(path):
existsd, w = self.client.exists_and_watch(path)
w.addCallback(node_watcher)
return existsd
def node_watcher(event):
self.assertEqual(event.type_name, "changed")
def verify_exists(node_stat):
self.assertTrue(node_stat)
return self.client.set("/rome", "magic")
d.addCallback(create_node)
d.addCallback(check_exists)
d.addCallback(verify_exists)
return d
def test_exists_with_watcher_and_close(self):
"""
Closing a connection with an watch outstanding behaves correctly.
"""
d = self.client.connect()
def node_watcher(event):
client = getattr(self, "client", None)
if client is not None and client.connected:
self.fail("Client should be disconnected")
def create_node(client):
return client.create("/syracuse")
def check_exists(path):
# shouldn't fire till unit test cleanup
d, w = self.client.exists_and_watch(path)
w.addCallback(node_watcher)
return d
def verify_exists(result):
self.assertTrue(result)
d.addCallback(create_node)
d.addCallback(check_exists)
d.addCallback(verify_exists)
return d
def test_exists_with_nonexistant_watcher(self):
"""
The exists method can also be used to set an optional watcher on a
node. The watch can be set on a node that does not yet exist.
"""
d = self.client.connect()
node_path = "/animals"
watcher_deferred = Deferred()
def create_container(path):
return self.client.create(node_path, "")
def check_exists(path):
exists, watch = self.client.exists_and_watch(
"%s/wooly-mammoth" % node_path)
watch.chainDeferred(watcher_deferred)
return exists
def new_connection(node_stat):
self.assertFalse(node_stat)
self.client2 = ZookeeperClient("127.0.0.1:2181")
return self.client2.connect()
def create_node(client):
self.assertEqual(client.connected, True)
return self.client2.create(
"%s/wooly-mammoth" % node_path, "extinct")
def shim(path):
return watcher_deferred
def verify_watch(event):
self.assertEqual(event.path, "%s/wooly-mammoth" % node_path)
self.assertEqual(event.type, zookeeper.CREATED_EVENT)
d.addCallback(create_container)
d.addCallback(check_exists)
d.addCallback(new_connection)
d.addCallback(create_node)
d.addCallback(shim)
d.addCallback(verify_watch)
return d
def test_create_sequence_node(self):
"""
The client can create a monotonically increasing sequence nodes.
"""
d = self.client.connect()
def create_node(client):
return self.client.create("/seq-a")
def create_seq_node(path):
return self.client.create(
"/seq-a/seq-", flags=zookeeper.EPHEMERAL | zookeeper.SEQUENCE)
def get_children(path):
return self.client.get_children("/seq-a")
def verify_children(children):
self.assertEqual(children, ["seq-0000000000", "seq-0000000001"])
d.addCallback(create_node)
d.addCallback(create_seq_node)
d.addCallback(create_seq_node)
d.addCallback(get_children)
d.addCallback(verify_children)
return d
def test_create_duplicate_node(self):
"""
Attempting to create a node that already exists results in a failure.
"""
d = self.client.connect()
def create_node(client):
return self.client.create("/abc")
def create_duplicate(path):
return self.client.create("/abc")
def verify_fails(*args):
self.fail("Invoked Callback")
def verify_succeeds(failure):
self.assertTrue(failure)
self.assertEqual(failure.value.args, ("node exists",))
d.addCallback(create_node)
d.addCallback(create_duplicate)
d.addCallback(verify_fails)
d.addErrback(verify_succeeds)
return d
def test_delete_nonexistant_node(self):
"""
Attempting to delete a node that already exists results in a failure.
"""
d = self.client.connect()
def delete_node(client):
return client.delete("/abcd")
def verify_fails(*args):
self.fail("Invoked Callback")
def verify_succeeds(failure):
self.assertTrue(failure)
self.assertEqual(
failure.value.args, ("no node /abcd",))
d.addCallback(delete_node)
d.addCallback(verify_fails)
d.addErrback(verify_succeeds)
return d
def test_set(self):
"""
The client can be used to set contents of a node.
"""
d = self.client.connect()
def create_node(client):
return client.create("/zebra", "horse")
def set_node(path):
return self.client.set("/zebra", "mammal")
def verify_contents(junk):
self.assertEqual(zookeeper.get(self.client.handle, "/zebra")[0],
"mammal")
d.addCallback(create_node)
d.addCallback(set_node)
d.addCallback(verify_contents)
return d
def test_set_nonexistant(self):
"""
if the client is used to set the contents of a nonexistant node
an error is raised.
"""
d = self.client.connect()
def set_node(client):
return client.set("/xy1")
def verify_fails(*args):
self.fail("Invoked Callback")
def verify_succeeds(failure):
self.assertTrue(failure)
self.assertTrue(
failure.value.args, ("no node /xy1"))
d.addCallback(set_node)
d.addCallback(verify_fails)
d.addErrback(verify_succeeds)
return d
def test_get_children(self):
d = self.client.connect()
def create_nodes(client):
zookeeper.create(
self.client.handle, "/tower", "", [PUBLIC_ACL], 0)
zookeeper.create(
self.client.handle, "/tower/london", "", [PUBLIC_ACL], 0)
zookeeper.create(
self.client.handle, "/tower/paris", "", [PUBLIC_ACL], 0)
return client
def get_children(client):
return client.get_children("/tower")
def verify_children(children):
self.assertEqual(children, ["paris", "london"])
d.addCallback(create_nodes)
d.addCallback(get_children)
d.addCallback(verify_children)
return d
def test_get_children_with_error(self):
"""If the result of an api call is an error, its propgated.
"""
d = self.client.connect()
def get_children(client):
# Get the children of a nonexistant node
return client.get_children("/tower")
def verify_failure(failure):
self.assertTrue(isinstance(failure, Failure))
self.assertTrue(
isinstance(failure.value, zookeeper.NoNodeException))
d.addCallback(get_children)
d.addBoth(verify_failure)
return d
# seems to be a segfault on this one, must be running latest zk
def test_get_children_with_watch(self):
"""
The get_children method optionally takes a watcher callable which will
be notified when the node is modified, or a child deleted or added.
"""
d = self.client.connect()
watch_deferred = Deferred()
def create_node(client):
return client.create("/jupiter")
def get_children(path):
ids, watch = self.client.get_children_and_watch(path)
watch.chainDeferred(watch_deferred)
return ids
def new_connection(children):
self.assertFalse(children)
self.client2 = ZookeeperClient("127.0.0.1:2181")
return self.client2.connect()
def trigger_watch(client):
zookeeper.create(
self.client2.handle, "/jupiter/io", "", [PUBLIC_ACL], 0)
return watch_deferred
def verify_observed(data):
self.assertTrue(data)
d.addCallback(create_node)
d.addCallback(get_children)
d.addCallback(new_connection)
d.addCallback(trigger_watch)
d.addCallback(verify_observed)
return d
def test_get_children_with_watch_container_deleted(self):
"""
Establishing a child watch on a path, and then deleting the path,
will fire a child event watch on the container. This seems a little
counterintutive, but zookeeper docs state they do this as a signal
the container will never have any children. And logically you'd
would want to fire, so that in case the container node gets recreated
later and the watch fires, you don't want to the watch to fire then,
as its a technically a different container.
"""
d = self.client.connect()
watch_deferred = Deferred()
def create_node(client):
return self.client.create("/prison")
def get_children(path):
childd, w = self.client.get_children_and_watch(path)
w.addCallback(verify_watch)
return childd
def delete_node(children):
return self.client.delete("/prison")
def verify_watch(event):
self.assertTrue(event.type_name, "child")
watch_deferred.callback(None)
d.addCallback(create_node)
d.addCallback(get_children)
d.addCallback(delete_node)
return watch_deferred
test_get_children_with_watch_container_deleted.timeout = 5
def test_get_no_children(self):
"""
Getting children of a node without any children returns an empty list.
"""
d = self.client.connect()
def create_node(client):
return self.client.create("/tower")
def get_children(path):
return self.client.get_children(path)
def verify_children(children):
self.assertEqual(children, [])
d.addCallback(create_node)
d.addCallback(get_children)
d.addCallback(verify_children)
return d
def test_get_children_nonexistant(self):
"""
Getting children of a nonexistant node raises a no node exception.
"""
d = self.client.connect()
def get_children(client):
return client.get_children("/tower")
d.addCallback(get_children)
self.failUnlessFailure(d, zookeeper.NoNodeException)
return d
def test_add_auth(self):
"""
The connection can have zero or more authentication infos. This
authentication infos are used when accessing nodes to veriy access
against the node's acl.
"""
d = self.client.connect()
credentials = "mary:apples"
user, password = credentials.split(":")
identity = "%s:%s" % (
user,
base64.b64encode(hashlib.new('sha1', credentials).digest()))
acl = {'id': identity, 'scheme': 'digest', 'perms': zookeeper.PERM_ALL}
failed = []
def add_auth_one(client):
d = client.add_auth("digest", "bob:martini")
# a little hack to avoid slowness around adding auth
# see https://issues.apache.org/jira/browse/ZOOKEEPER-770
# by pushing an additional message send/response cycle
# we don't have to wait for the io thread to timeout
# on the socket.
client.exists("/orchard")
return d
def create_node(client):
return client.create("/orchard", "apple trees", acls=[acl])
def try_node_access(path):
return self.client.set("/orchard", "bar")
def node_access_failed(failure):
self.assertEqual(failure.value.args, ("not authenticated /orchard",))
failed.append(True)
return
def add_auth_two(result):
d = self.client.add_auth("digest", credentials)
# a little hack to avoid slowness around adding auth
# see https://issues.apache.org/jira/browse/ZOOKEEPER-770
self.client.get_children("/orchard")
return d
def verify_node_access(stat):
self.assertEqual(stat['version'], 1)
self.assertEqual(stat['dataLength'], 3)
self.assertTrue(failed) # we should have hit the errback
d.addCallback(add_auth_one)
d.addCallback(create_node)
d.addCallback(try_node_access)
d.addErrback(node_access_failed)
d.addCallback(add_auth_two)
d.addCallback(try_node_access)
d.addCallback(verify_node_access)
return d
def test_add_auth_with_error(self):
"""
On add_auth error the deferred errback is invoked with the exception.
"""
d = self.client.connect()
def _fake_auth(handle, scheme, identity, callback):
callback(0, zookeeper.AUTHFAILED)
return 0
mock_auth = self.mocker.replace("zookeeper.add_auth")
mock_auth(ANY, ANY, ANY, ANY)
self.mocker.call(_fake_auth)
self.mocker.replay()
def add_auth(client):
d = self.client.add_auth("digest", "mary:lamb")
return d
def verify_failure(failure):
self.assertTrue(
isinstance(failure.value, zookeeper.AuthFailedException))
def assert_failed(result):
self.fail("should not get here")
d.addCallback(add_auth)
d.addCallback(assert_failed)
d.addErrback(verify_failure)
return d
def test_set_acl(self):
"""
The client can be used to set an ACL on a node.
"""
d = self.client.connect()
acl = [PUBLIC_ACL,
dict(scheme="digest",
id="zebra:moon",
perms=zookeeper.PERM_ALL)]
def create_node(client):
return client.create("/moose")
def set_acl(path):
return self.client.set_acl(path, acl)
def verify_acl(junk):
self.assertEqual(
zookeeper.get_acl(self.client.handle, "/moose")[1],
acl)
d.addCallback(create_node)
d.addCallback(set_acl)
d.addCallback(verify_acl)
return d
def test_set_acl_with_error(self):
"""
on error set_acl invokes the deferred's errback with an exception.
"""
d = self.client.connect()
acl = dict(scheme="digest", id="a:b", perms=zookeeper.PERM_ALL)
def set_acl(client):
return client.set_acl("/zebra-moon22", [acl])
def verify_failure(failure):
self.assertTrue(
isinstance(failure.value, zookeeper.NoNodeException))
d.addCallback(set_acl)
d.addErrback(verify_failure)
return d
def test_get_acl(self):
"""
The client can be used to get an ACL on a node.
"""
d = self.client.connect()
def create_node(client):
return client.create("/moose")
def get_acl(path):
return self.client.get_acl(path)
def verify_acl((acls, stat)):
self.assertEqual(acls, [PUBLIC_ACL])
d.addCallback(create_node)
d.addCallback(get_acl)
d.addCallback(verify_acl)
return d
def test_get_acl_error(self):
"""
On error the acl callback invokes the deferred errback with the
exception.
"""
d = self.client.connect()
def inject_error(result, d):
error = zookeeper.ZooKeeperException()
d.errback(error)
return error
def get_acl(path):
# Get the ACL of a nonexistant node
return self.client.get_acl("/moose")
def verify_failure(failure):
self.assertTrue(isinstance(failure, Failure))
self.assertTrue(
isinstance(failure.value, zookeeper.ZooKeeperException))
d.addCallback(get_acl)
d.addBoth(verify_failure)
return d
def test_client_id(self):
"""
The client exposes a client id which is useful when examining
the server logs.
"""
# if we're not connected returns none
self.assertEqual(self.client.client_id, None)
d = self.client.connect()
def verify_client_id(client):
self.assertTrue(isinstance(self.client.client_id, tuple))
self.assertTrue(isinstance(self.client.client_id[0], long))
self.assertTrue(isinstance(self.client.client_id[1], str))
d.addCallback(verify_client_id)
return d
def test_sync(self):
"""
The sync method on the client flushes the connection to leader.
In practice this seems hard to test functionally, but we at
least verify the method executes without issue.
"""
d = self.client.connect()
def create_node(client):
return client.create("/abc")
def client_sync(path):
return self.client.sync(path)
def verify_sync(result):
self.assertTrue(
zookeeper.exists(self.client.handle, "/abc"))
d.addCallback(create_node)
d.addCallback(client_sync)
d.addCallback(verify_sync)
return d
def test_property_servers(self):
"""
The servers property of the client, shows which if any servers
it might be connected, else it returns.
"""
self.assertEqual(self.client.servers, None)
d = self.client.connect()
def verify_servers(client):
self.assertEqual(client.servers, "127.0.0.1:2181")
d.addCallback(verify_servers)
return d
def test_property_session_timeout(self):
"""
The negotiated session timeout is available as a property on the
client. If the client isn't connected, the value is None.
"""
self.assertEqual(self.client.session_timeout, None)
d = self.client.connect()
def verify_session_timeout(client):
self.assertIn(client.session_timeout, (4000, 10000))
d.addCallback(verify_session_timeout)
return d
def test_property_unrecoverable(self):
"""
The unrecoverable property specifies whether the connection can be
recovered or must be discarded.
"""
d = self.client.connect()
def verify_recoverable(client):
self.assertEqual(client.unrecoverable, False)
return client
d.addCallback(verify_recoverable)
return d
def test_invalid_watcher(self):
"""
Setting an invalid watcher raises a syntaxerror.
"""
d = self.client.connect()
def set_invalid_watcher(client):
return client.set_connection_watcher(1)
def verify_invalid(failure):
self.assertEqual(failure.value.args, ("Invalid Watcher 1",))
self.assertTrue(isinstance(failure.value, SyntaxError))
d.addCallback(set_invalid_watcher)
d.addErrback(verify_invalid)
return d
def test_connect_with_server(self):
"""
A client's servers can be specified in the connect method.
"""
d = self.client.connect("127.0.0.1:2181")
def verify_connected(client):
self.assertTrue(client.connected)
d.addCallback(verify_connected)
return d
def test_connect_with_error(self):
"""
An error in the connect invokes the deferred errback with exception.
"""
def _fake_init(handle, callback, timeout):
callback(0, 0, zookeeper.ASSOCIATING_STATE, "")
return 0
mock_init = self.mocker.replace("zookeeper.init")
mock_init(ANY, ANY, ANY)
self.mocker.call(_fake_init)
self.mocker.replay()
d = self.client.connect()
def verify_error(failure):
self.assertFalse(self.client.connected)
self.assertTrue(isinstance(failure.value, ConnectionException))
self.assertEqual(failure.value.args[0], "connection error")
def assert_failed(any):
self.fail("should not be invoked")
d.addCallback(assert_failed)
d.addErrback(verify_error)
return d
test_connect_with_error.timeout = 5
def test_connect_timeout(self):
"""
A timeout in seconds can be specified on connect, if the client hasn't
connected before then, then an errback is invoked with a timeout
exception.
"""
# Connect to a non standard port with nothing at the remote side.
d = self.client.connect("127.0.0.1:2182", timeout=0.2)
def verify_timeout(failure):
self.assertTrue(
isinstance(failure.value, ConnectionTimeoutException))
def assert_failure(any):
self.fail("should not be reached")
d.addCallback(assert_failure)
d.addErrback(verify_timeout)
return d
def test_connect_ensured(self):
"""
All of the client apis (with the exception of connect) attempt
to ensure the client is connected before executing an operation.
"""
self.assertFailure(
self.client.get_children("/abc"), zookeeper.ZooKeeperException)
self.assertFailure(
self.client.create("/abc"), zookeeper.ZooKeeperException)
self.assertFailure(
self.client.set("/abc", "123"), zookeeper.ZooKeeperException)
def test_connect_multiple_raises(self):
"""
Attempting to connect on a client that is already connected raises
an exception.
"""
d = self.client.connect()
def connect_again(client):
d = client.connect()
self.failUnlessFailure(d, zookeeper.ZooKeeperException)
return d
d.addCallback(connect_again)
return d
def test_bad_result_raises_error(self):
"""
A not OK return from zookeeper api method result raises an exception.
"""
mock_acreate = self.mocker.replace("zookeeper.acreate")
mock_acreate(ANY, ANY, ANY, ANY, ANY, ANY)
self.mocker.result(-100)
self.mocker.replay()
d = self.client.connect()
def verify_failure(client):
d = client.create("/abc")
self.failUnlessFailure(d, zookeeper.ZooKeeperException)
d.addCallback(verify_failure)
return d
def test_connection_watcher(self):
"""
A connection watcher can be set that receives notices on when
the connection state changes. Technically zookeeper would also
use this as a global watcher for node watches, but zkpython
doesn't expose that api, as its mostly considered legacy.
its out of scope to simulate a connection level event within unit tests
such as the server restarting.
"""
d = self.client.connect()
observed = []
def watch(*args):
observed.append(args)
def set_global_watcher(client):
client.set_connection_watcher(watch)
return client
def close_connection(client):
return client.close()
def verify_observed(stat):
self.assertFalse(observed)
d.addCallback(set_global_watcher)
d.addCallback(close_connection)
d.addCallback(verify_observed)
return d
def test_close_not_connected(self):
"""
If the client is not connected, closing returns None.
"""
self.assertEqual(self.client.close(), None)
def test_invalid_connection_error_callback(self):
self.assertRaises(TypeError,
self.client.set_connection_error_callback,
None)
def test_invalid_session_callback(self):
self.assertRaises(TypeError,
self.client.set_session_callback,
None)
txzookeeper-0.9.8/txzookeeper/tests/common.py 0000664 0001750 0001750 00000017442 12144707107 021731 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011, 2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import os
import os.path
import shutil
import subprocess
import tempfile
from itertools import chain
from collections import namedtuple
from glob import glob
ServerInfo = namedtuple(
"ServerInfo", "server_id client_port election_port leader_port")
class ManagedZooKeeper(object):
"""Class to manage the running of a ZooKeeper instance for testing.
Note: no attempt is made to probe the ZooKeeper instance is
actually available, or that the selected port is free. In the
future, we may want to do that, especially when run in a
Hudson/Buildbot context, to ensure more test robustness."""
def __init__(self, software_path, server_info, peers=()):
"""Define the ZooKeeper test instance.
@param install_path: The path to the install for ZK
@param port: The port to run the managed ZK instance
"""
self.install_path = software_path
self.server_info = server_info
self.host = "127.0.0.1"
self.peers = peers
self.working_path = tempfile.mkdtemp()
self._running = False
def run(self):
"""Run the ZooKeeper instance under a temporary directory.
Writes ZK log messages to zookeeper.log in the current directory.
"""
config_path = os.path.join(self.working_path, "zoo.cfg")
log_path = os.path.join(self.working_path, "log")
log4j_path = os.path.join(self.working_path, "log4j.properties")
data_path = os.path.join(self.working_path, "data")
# various setup steps
if not os.path.exists(self.working_path):
os.mkdir(self.working_path)
if not os.path.exists(log_path):
os.mkdir(log_path)
if not os.path.exists(data_path):
os.mkdir(data_path)
with open(config_path, "w") as config:
config.write("""
tickTime=1000
dataDir=%s
clientPort=%s
maxClientCnxns=0
maxSessionTimeout=5000
minSessionTimeout=2000
""" % (data_path, self.server_info.client_port))
# setup a replicated setup if peers are specified
if self.peers:
servers_cfg = []
for p in chain((self.server_info,), self.peers):
servers_cfg.append("server.%s=localhost:%s:%s" % (
p.server_id, p.leader_port, p.election_port))
with open(config_path, "a") as config:
config.write("""
initLimit=4
syncLimit=2
%s
""" % ("\n".join(servers_cfg)))
# Write server ids into datadir
with open(os.path.join(data_path, "myid"), "w") as myid_file:
myid_file.write(str(self.server_info.server_id))
with open(log4j_path, "w") as log4j:
log4j.write("""
# DEFAULT: console appender only
log4j.rootLogger=INFO, ROLLINGFILE
log4j.appender.ROLLINGFILE.layout=org.apache.log4j.PatternLayout
log4j.appender.ROLLINGFILE.layout.ConversionPattern=%d{ISO8601} [myid:%X{myid}] - %-5p [%t:%C{1}@%L] - %m%n
log4j.appender.ROLLINGFILE=org.apache.log4j.RollingFileAppender
log4j.appender.ROLLINGFILE.Threshold=DEBUG
log4j.appender.ROLLINGFILE.File=""" + (
self.working_path + os.sep + "zookeeper.log\n"))
self.process = subprocess.Popen(
args=["java",
"-cp", self.classpath,
"-Dzookeeper.log.dir=%s" % log_path,
"-Dzookeeper.root.logger=INFO,CONSOLE",
"-Dlog4j.configuration=file:%s" % log4j_path,
# "-Dlog4j.debug",
"org.apache.zookeeper.server.quorum.QuorumPeerMain",
config_path],
)
self._running = True
@property
def classpath(self):
"""Get the classpath necessary to run ZooKeeper."""
# Two possibilities, as seen in zkEnv.sh:
# Check for a release - top-level zookeeper-*.jar?
jars = glob((os.path.join(
self.install_path, 'zookeeper-*.jar')))
if jars:
# Relase build (`ant package`)
jars.extend(glob(os.path.join(
self.install_path,
"lib/*.jar")))
else:
# Development build (plain `ant`)
jars = glob((os.path.join(
self.install_path, 'build/zookeeper-*.jar')))
jars.extend(glob(os.path.join(
self.install_path,
"build/lib/*.jar")))
return ":".join(jars)
@property
def address(self):
"""Get the address of the ZooKeeper instance."""
return "%s:%s" % (self.host, self.client_port)
@property
def running(self):
return self._running
@property
def client_port(self):
return self.server_info.client_port
def reset(self):
"""Stop the zookeeper instance, cleaning out its on disk-data."""
self.stop()
shutil.rmtree(os.path.join(self.working_path, "data"))
os.mkdir(os.path.join(self.working_path, "data"))
with open(os.path.join(self.working_path, "data", "myid"), "w") as fh:
fh.write(str(self.server_info.server_id))
def stop(self):
"""Stop the Zookeeper instance, retaining on disk state."""
if not self._running:
return
self.process.terminate()
self.process.wait()
self._running = False
def destroy(self):
"""Stop the ZooKeeper instance and destroy its on disk-state"""
# called by at exit handler, reimport to avoid cleanup race.
import shutil
self.stop()
shutil.rmtree(self.working_path)
class ZookeeperCluster(object):
def __init__(self, install_path, size=3, port_offset=20000):
self._install_path = install_path
self._servers = []
# Calculate ports and peer group
port = port_offset
peers = []
for i in range(size):
port += i * 10
info = ServerInfo(i + 1, port, port + 1, port + 2)
peers.append(info)
# Instantiate Managed ZK Servers
for i in range(size):
server_peers = list(peers)
server_info = server_peers.pop(i)
self._servers.append(
ManagedZooKeeper(
self._install_path, server_info, server_peers))
def __getitem__(self, k):
return self._servers[k]
def __iter__(self):
return iter(self._servers)
def start(self):
# Zookeeper client expresses a preference for either lower ports or
# lexographical ordering of hosts, to ensure that all servers have a
# chance to startup, start them in reverse order.
for server in reversed(list(self)):
server.run()
# Giving the servers a moment to start, decreases the overall time
# required for a client to successfully connect (2s vs. 4s without
# the sleep).
import time
time.sleep(2)
def stop(self):
for server in self:
server.stop()
self._servers = []
def terminate(self):
for server in self:
server.destroy()
def reset(self):
for server in self:
server.reset()
txzookeeper-0.9.8/txzookeeper/tests/utils.py 0000664 0001750 0001750 00000002517 11745322401 021572 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import zookeeper
def deleteTree(path="/", handle=1):
"""
Destroy all the nodes in zookeeper (typically under a chroot for testing)
"""
for child in zookeeper.get_children(handle, path):
if child == "zookeeper": # skip the metadata node
continue
child_path = "/" + ("%s/%s" % (path, child)).strip("/")
try:
deleteTree(child_path, handle)
zookeeper.delete(handle, child_path, -1)
except zookeeper.ZooKeeperException, e:
print "Error on path", child_path, e
txzookeeper-0.9.8/txzookeeper/tests/proxy.py 0000664 0001750 0001750 00000005624 11745322401 021615 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from twisted.protocols import portforward
class Blockable(object):
_blocked = None
def set_blocked(self, value):
value = bool(value)
self._blocked = value
if self.transport and not self._blocked:
self.transport.resumeProducing()
class ProxyClient(portforward.ProxyClient, Blockable):
def dataReceived(self, data):
if self._blocked:
return
portforward.ProxyClient.dataReceived(self, data)
def setServer(self, server):
server.set_blocked(self._blocked)
super(ProxyClient, self).setServer(server)
def connectionMade(self):
self.peer.setPeer(self)
if not self._blocked:
# The server waits till the client is connected
self.peer.transport.resumeProducing()
else:
self.transport.pauseProducing()
class ProxyClientFactory(portforward.ProxyClientFactory):
protocol = ProxyClient
class ProxyServer(portforward.ProxyServer, Blockable):
clientProtocolFactory = ProxyClientFactory
def dataReceived(self, data):
if self._blocked:
return
portforward.ProxyServer.dataReceived(self, data)
class ProxyFactory(portforward.ProxyFactory):
protocol = ProxyServer
instance = _blocked = False
def lose_connection(self):
"""Terminate both ends of the proxy connection."""
if self.instance:
self.instance.transport.loseConnection()
if self.instance.peer:
self.instance.peer.transport.loseConnection()
def set_blocked(self, value):
self._blocked = bool(value)
if self.instance:
self.instance.set_blocked(self._blocked)
if self.instance.peer:
self.instance.peer.set_blocked(self._blocked)
def buildProtocol(self, addr):
# Track last protocol used, on reconnect any pauses are disabled.
self.instance = portforward.ProxyFactory.buildProtocol(self, addr)
# Propogate the value, the client will aggressively try to
# reconnect else.
self.instance.set_blocked(self._blocked)
return self.instance
txzookeeper-0.9.8/txzookeeper/client.py 0000664 0001750 0001750 00000072756 12144716331 020565 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
from collections import namedtuple
from functools import partial
from twisted.internet import defer, reactor
import zookeeper
# Default session timeout
DEFAULT_SESSION_TIMEOUT = 10000
# Default node acl (public)
ZOO_OPEN_ACL_UNSAFE = {
"perms": zookeeper.PERM_ALL,
"scheme": "world",
"id": "anyone"}
# Skip acls for policy objects @ a higher level
SKIP_ACLS = object()
# Map result codes to exceptions classes.
ERROR_MAPPING = {
zookeeper.APIERROR: zookeeper.ApiErrorException,
zookeeper.AUTHFAILED: zookeeper.AuthFailedException,
zookeeper.BADARGUMENTS: zookeeper.BadArgumentsException,
zookeeper.BADVERSION: zookeeper.BadVersionException,
zookeeper.CLOSING: zookeeper.ClosingException,
zookeeper.CONNECTIONLOSS: zookeeper.ConnectionLossException,
zookeeper.DATAINCONSISTENCY: zookeeper.DataInconsistencyException,
zookeeper.INVALIDACL: zookeeper.InvalidACLException,
zookeeper.INVALIDCALLBACK: zookeeper.InvalidCallbackException,
zookeeper.INVALIDSTATE: zookeeper.InvalidStateException,
zookeeper.MARSHALLINGERROR: zookeeper.MarshallingErrorException,
zookeeper.NOAUTH: zookeeper.NoAuthException,
zookeeper.NOCHILDRENFOREPHEMERALS: (
zookeeper.NoChildrenForEphemeralsException),
zookeeper.NONODE: zookeeper.NoNodeException,
zookeeper.NODEEXISTS: zookeeper.NodeExistsException,
zookeeper.NOTEMPTY: zookeeper.NotEmptyException,
zookeeper.NOTHING: zookeeper.NothingException,
zookeeper.OPERATIONTIMEOUT: zookeeper.OperationTimeoutException,
zookeeper.RUNTIMEINCONSISTENCY: zookeeper.RuntimeInconsistencyException,
zookeeper.SESSIONEXPIRED: zookeeper.SessionExpiredException,
zookeeper.SESSIONMOVED: zookeeper.SessionMovedException,
zookeeper.SYSTEMERROR: zookeeper.SystemErrorException,
zookeeper.UNIMPLEMENTED: zookeeper.UnimplementedException}
# Mapping of connection state values to human strings.
STATE_NAME_MAPPING = {
zookeeper.ASSOCIATING_STATE: "associating",
zookeeper.AUTH_FAILED_STATE: "auth-failed",
zookeeper.CONNECTED_STATE: "connected",
zookeeper.CONNECTING_STATE: "connecting",
zookeeper.EXPIRED_SESSION_STATE: "expired",
None: "unknown",
}
# Mapping of event type to human string.
TYPE_NAME_MAPPING = {
zookeeper.NOTWATCHING_EVENT: "not-watching",
zookeeper.SESSION_EVENT: "session",
zookeeper.CREATED_EVENT: "created",
zookeeper.DELETED_EVENT: "deleted",
zookeeper.CHANGED_EVENT: "changed",
zookeeper.CHILD_EVENT: "child",
None: "unknown",
}
class NotConnectedException(zookeeper.ZooKeeperException):
"""
Raised if an attempt is made to use the client's api before the
client has a connection to a zookeeper server.
"""
class ConnectionException(zookeeper.ZooKeeperException):
"""
Raised if an error occurs during the client's connection attempt.
"""
@property
def state_name(self):
return STATE_NAME_MAPPING[self.args[2]]
@property
def type_name(self):
return TYPE_NAME_MAPPING[self.args[1]]
@property
def handle(self):
return self.args[3]
def __str__(self):
return (
""
% (self.handle, self.type_name, self.state_name))
def is_connection_exception(e):
"""
For connection errors in response to api calls, a utility method
to determine if the cause is a connection exception.
"""
return isinstance(e,
(zookeeper.ClosingException,
zookeeper.ConnectionLossException,
zookeeper.SessionExpiredException))
class ConnectionTimeoutException(zookeeper.ZooKeeperException):
"""
An exception raised when the we can't connect to zookeeper within
the user specified timeout period.
"""
class ClientEvent(namedtuple("ClientEvent",
'type, connection_state, path, handle')):
"""
A client event is returned when a watch deferred fires. It denotes
some event on the zookeeper client that the watch was requested on.
"""
@property
def type_name(self):
return TYPE_NAME_MAPPING[self.type]
@property
def state_name(self):
return STATE_NAME_MAPPING[self.connection_state]
def __repr__(self):
return "" % (
self.type_name, self.path, self.state_name, self.handle)
class ZookeeperClient(object):
"""Asynchronous twisted client for zookeeper."""
def __init__(self, servers=None, session_timeout=None):
"""
@param servers: A string specifying the servers and their
ports to connect to. Multiple servers can be
specified in comma separated fashion. if they are,
then the client will automatically rotate
among them if a server connection fails. Optionally
a chroot can be specified. A full server spec looks
like host:port/chroot_path
@param session_timeout: The client's zookeeper session timeout can be
hinted. The actual value is negotiated between the
client and server based on their respective
configurations.
"""
self._servers = servers
self._session_timeout = session_timeout
self._session_event_callback = None
self._connection_error_callback = None
self.connected = False
self.handle = None
def __repr__(self):
if not self.client_id:
session_id = ""
else:
session_id = self.client_id[0]
return "<%s session: %s handle: %r state: %s>" % (
self.__class__.__name__, session_id, self.handle, self.state)
def _check_connected(self, d):
if not self.connected:
d.errback(NotConnectedException("not connected"))
return d
def _check_result(self, result_code, deferred, extra_codes=(), path=None):
"""Check an API call or result for errors.
:param result_code: The api result code.
:param deferred: The deferred returned the client api consumer.
:param extra_codes: Additional result codes accepted as valid/ok.
If the result code is an error, an appropriate Exception class
is constructed and the errback on the deferred is invoked with it.
"""
error = None
if not result_code == zookeeper.OK and not result_code in extra_codes:
error_msg = zookeeper.zerror(result_code)
if path is not None:
error_msg += " %s" % path
error_class = ERROR_MAPPING.get(
result_code, zookeeper.ZooKeeperException)
error = error_class(error_msg)
if is_connection_exception(error):
# Route connection errors to a connection level error
# handler if specified.
if self._connection_error_callback:
# The result of the connection error handler is returned
# to the api invoker.
d = defer.maybeDeferred(
self._connection_error_callback,
self, error)
d.chainDeferred(deferred)
return True
deferred.errback(error)
return True
return None
def _get(self, path, watcher):
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_get(result_code, value, stat):
if self._check_result(result_code, d, path=path):
return
d.callback((value, stat))
callback = self._zk_thread_callback(_cb_get)
watcher = self._wrap_watcher(watcher, "get", path)
result = zookeeper.aget(self.handle, path, watcher, callback)
self._check_result(result, d, path=path)
return d
def _get_children(self, path, watcher):
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_get_children(result_code, children):
if self._check_result(result_code, d, path=path):
return
d.callback(children)
callback = self._zk_thread_callback(_cb_get_children)
watcher = self._wrap_watcher(watcher, "child", path)
result = zookeeper.aget_children(self.handle, path, watcher, callback)
self._check_result(result, d, path=path)
return d
def _exists(self, path, watcher):
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_exists(result_code, stat):
if self._check_result(
result_code, d, extra_codes=(zookeeper.NONODE,), path=path):
return
d.callback(stat)
callback = self._zk_thread_callback(_cb_exists)
watcher = self._wrap_watcher(watcher, "exists", path)
result = zookeeper.aexists(self.handle, path, watcher, callback)
self._check_result(result, d, path=path)
return d
def _wrap_watcher(self, watcher, watch_type, path):
if watcher is None:
return watcher
if not callable(watcher):
raise SyntaxError("invalid watcher")
return self._zk_thread_callback(
partial(self._session_event_wrapper, watcher))
def _session_event_wrapper(self, watcher, event_type, conn_state, path):
"""Watch wrapper that diverts session events to a connection callback.
"""
# If it's a session event pass it to the session callback, else
# ignore it. Session events are sent repeatedly to watchers
# which we have modeled after deferred, which only accept a
# single return value.
if event_type == zookeeper.SESSION_EVENT:
if self._session_event_callback:
self._session_event_callback(
self, ClientEvent(
event_type, conn_state, path, self.handle))
# We do propagate to watch deferreds, in one case in
# particular, namely if the session is expired, in which
# case the watches are dead, and we send an appropriate
# error.
if conn_state == zookeeper.EXPIRED_SESSION_STATE:
error = zookeeper.SessionExpiredException("Session expired")
return watcher(None, None, None, error=error)
else:
return watcher(event_type, conn_state, path)
def _zk_thread_callback(self, func, *f_args, **f_kw):
"""
The client library invokes callbacks in a separate thread, we wrap
any user defined callback so that they are called back in the main
thread after, zookeeper calls the wrapper.
"""
f_args = list(f_args)
def wrapper(handle, *args): # pragma: no cover
# make a copy, the conn watch callback gets invoked multiple times
cb_args = list(f_args)
cb_args.extend(args)
reactor.callFromThread(func, *cb_args, **f_kw)
return wrapper
@property
def servers(self):
"""
Servers that we're connected to or None if the client is not connected
"""
if self.connected:
return self._servers
@property
def session_timeout(self):
"""
The negotiated session timeout for this connection, in milliseconds.
If the client is not connected the value is None.
"""
if self.connected:
return zookeeper.recv_timeout(self.handle)
@property
def state(self):
"""
What's the current state of this connection, result is an
integer value corresponding to zoookeeper module constants.
"""
if self.connected:
return zookeeper.state(self.handle)
@property
def client_id(self):
"""Returns the client id that identifies the server side session.
A client id is a tuple represented by the session id and
session password. It can be used to manually connect to an
extant server session (which contains associated ephemeral
nodes and watches)/ The connection's client id is also useful
when introspecting the server logs for specific client
activity.
"""
if self.handle is None:
return None
try:
return zookeeper.client_id(self.handle)
# Invalid handle
except zookeeper.ZooKeeperException:
return None
@property
def unrecoverable(self):
"""
Boolean value representing whether the current connection can be
recovered.
"""
try:
return bool(zookeeper.is_unrecoverable(self.handle))
except zookeeper.ZooKeeperException:
# guard against invalid handles
return True
def add_auth(self, scheme, identity):
"""Adds an authentication identity to this connection.
A connection can use multiple authentication identities at the
same time, all are checked when verifying acls on a node.
@param scheme: a string specifying a an authentication scheme
valid values include 'digest'.
@param identity: a string containing username and password colon
separated, for example 'mary:apples'
"""
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_authenticated(result_code):
if self._check_result(result_code, d):
return
d.callback(self)
callback = self._zk_thread_callback(_cb_authenticated)
result = zookeeper.add_auth(self.handle, scheme, identity, callback)
self._check_result(result, d)
return d
def close(self, force=False):
"""
Close the underlying socket connection and server side session.
@param force: boolean, require the connection to be closed now or
an exception be raised.
"""
self.connected = False
if self.handle is None:
return
try:
result = zookeeper.close(self.handle)
except zookeeper.ZooKeeperException:
self.handle = None
return
d = defer.Deferred()
if self._check_result(result, d):
return d
self.handle = None
d.callback(True)
return d
def connect(self, servers=None, timeout=10, client_id=None):
"""
Establish a connection to the given zookeeper server(s).
@param servers: A string specifying the servers and their ports to
connect to. Multiple servers can be specified in
comma separated fashion.
@param timeout: How many seconds to wait on a connection to the
zookeeper servers.
@param session_id:
@returns A deferred that's fired when the connection is established.
"""
d = defer.Deferred()
if self.connected:
return defer.fail(
zookeeper.ZooKeeperException("Already Connected"))
# Use a scheduled function to ensure a timeout.
def _check_timeout():
# Close the handle
try:
if self.handle is not None:
zookeeper.close(self.handle)
except zookeeper.ZooKeeperException:
pass
d.errback(
ConnectionTimeoutException("could not connect before timeout"))
scheduled_timeout = reactor.callLater(timeout, _check_timeout)
# Assemble an on connect callback with closure variable access.
callback = partial(self._cb_connected, scheduled_timeout, d)
callback = self._zk_thread_callback(callback)
if self._session_timeout is None:
self._session_timeout = DEFAULT_SESSION_TIMEOUT
if servers is not None:
self._servers = servers
# Use client id if specified.
if client_id:
self.handle = zookeeper.init(
self._servers, callback, self._session_timeout, client_id)
else:
self.handle = zookeeper.init(
self._servers, callback, self._session_timeout)
return d
def _cb_connected(
self, scheduled_timeout, connect_deferred, type, state, path):
"""This callback is invoked through the lifecycle of the connection.
It's used for all connection level events and session events.
"""
# Cancel the timeout delayed task if it hasn't fired.
if scheduled_timeout.active():
scheduled_timeout.cancel()
# Update connected boolean
if state == zookeeper.CONNECTED_STATE:
self.connected = True
elif state != zookeeper.CONNECTING_STATE:
self.connected = False
if connect_deferred.called:
# If we timed out and then connected, then close the conn.
if state == zookeeper.CONNECTED_STATE and scheduled_timeout.called:
self.close()
self.handle = -1
return
# Send session events to the callback, in addition to any
# duplicate session events that will be sent for extant watches.
if self._session_event_callback:
self._session_event_callback(
self, ClientEvent(type, state, path, self.handle))
return
# Connected successfully, or If we're expired on an initial
# connect, someone else expired us.
elif state in (zookeeper.CONNECTED_STATE,
zookeeper.EXPIRED_SESSION_STATE):
connect_deferred.callback(self)
return
connect_deferred.errback(
ConnectionException("connection error", type, state, path))
def create(self, path, data="", acls=[ZOO_OPEN_ACL_UNSAFE], flags=0):
"""
Create a node with the given data and access control.
@params path: The path to the node
@params data: The node's content
@params acls: A list of dictionaries specifying permissions.
@params flags: Node creation flags (ephemeral, sequence, persistent)
"""
if acls == SKIP_ACLS:
acls = [ZOO_OPEN_ACL_UNSAFE]
d = defer.Deferred()
if self._check_connected(d):
return d
callback = self._zk_thread_callback(
self._cb_created, d, data, acls, flags)
result = zookeeper.acreate(
self.handle, path, data, acls, flags, callback)
self._check_result(result, d, path=path)
return d
def _cb_created(self, d, data, acls, flags, result_code, path):
if self._check_result(result_code, d, path=path):
return
d.callback(path)
def delete(self, path, version=-1):
"""
Delete the node at the given path. If the current node version on the
server is more recent than that supplied by the client, a bad version
exception wil be thrown. A version of -1 (default) specifies any
version.
@param path: the path of the node to be deleted.
@param version: the integer version of the node.
"""
d = defer.Deferred()
callback = self._zk_thread_callback(self._cb_deleted, d, path)
result = zookeeper.adelete(self.handle, path, version, callback)
self._check_result(result, d, path=path)
return d
def _cb_deleted(self, d, path, result_code):
if self._check_result(result_code, d, path=path):
return
d.callback(result_code)
def exists(self, path):
"""
Check that the given node path exists. Returns a deferred that
holds the node stat information if the node exists (created,
modified, version, etc.), or ``None`` if it does not exist.
@param path: The path of the node whose existence will be checked.
"""
return self._exists(path, None)
def exists_and_watch(self, path):
"""
Check that the given node path exists and set watch.
In addition to the deferred method result, this method returns
a deferred that is called back when the node is modified or
removed (once).
@param path: The path of the node whose existence will be checked.
"""
d = defer.Deferred()
def watcher(event_type, conn_state, path, error=None):
if error:
d.errback(error)
else:
d.callback(ClientEvent(
event_type, conn_state, path, self.handle))
return self._exists(path, watcher), d
def get(self, path):
"""
Get the node's data for the given node path. Returns a
deferred that holds the content of the node.
@param path: The path of the node whose content will be retrieved.
"""
return self._get(path, None)
def get_and_watch(self, path):
"""
Get the node's data for the given node path and set watch.
In addition to the deferred method result, this method returns
a deferred that is called back when the node is modified or
removed (once).
@param path: The path of the node whose content will be retrieved.
"""
d = defer.Deferred()
def watcher(event_type, conn_state, path, error=None):
if error:
d.errback(error)
else:
d.callback(ClientEvent(
event_type, conn_state, path, self.handle))
return self._get(path, watcher), d
def get_children(self, path):
"""
Get the ids of all children directly under the given path.
@param path: The path of the node whose children will be retrieved.
"""
return self._get_children(path, None)
def get_children_and_watch(self, path):
"""
Get the ids of all children directly under the given path.
In addition to the deferred method result, this method returns
a deferred that is called back when a change happens on the
provided path (once).
@param path: The path of the node whose children will be retrieved.
"""
d = defer.Deferred()
def watcher(event_type, conn_state, path, error=None):
if error:
d.errback(error)
else:
d.callback(ClientEvent(
event_type, conn_state, path, self.handle))
return self._get_children(path, watcher), d
def get_acl(self, path):
"""
Get the list of acls that apply to node with the give path.
Each acl is a dictionary containing keys/values for scheme, id,
and perms.
@param path: The path of the node whose acl will be retrieved.
"""
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_get_acl(result_code, acls, stat):
if self._check_result(result_code, d, path=path):
return
d.callback((acls, stat))
callback = self._zk_thread_callback(_cb_get_acl)
result = zookeeper.aget_acl(self.handle, path, callback)
self._check_result(result, d, path=path)
return d
def set_acl(self, path, acls, version=-1):
"""
Set the list of acls on a node.
Each acl is a dictionary containing keys/values for scheme, id,
and perms. The value for id is username:hash_value The hash_value
component is the base64 encoded sha1 hash of a username and
password that's colon separated. For example
>>> import hashlib, base64
>>> digest = base64.b64encode(
... hashlib.new('sha1', 'mary:apples').digest()))
>>> id = '%s:%s'%('mary', digest)
>>> id
'mary:9MTr9XuZvmudebp9aOo4DtXwyII='
>>> acl = {'id':id, 'scheme':'digest', 'perms':zookeeper.PERM_ALL}
@param path: The string path to the node.
@param acls: A list of acl dictionaries.
@param version: A version id of the node we're modifying, if this
doesn't match the version on the server, then a
BadVersionException is raised.
"""
d = defer.Deferred()
if self._check_connected(d):
return d
callback = self._zk_thread_callback(self._cb_set_acl, d, path, acls)
result = zookeeper.aset_acl(
self.handle, path, version, acls, callback)
self._check_result(result, d, path=path)
return d
def _cb_set_acl(self, d, path, acls, result_code):
if self._check_result(result_code, d, path=path):
return
d.callback(result_code)
def set(self, path, data="", version=-1):
"""
Sets the data of a node at the given path. If the current node version
on the server is more recent than that supplied by the client, a bad
version exception wil be thrown. A version of -1 (default) specifies
any version.
@param path: The path of the node whose data we will set.
@param data: The data to store on the node.
@param version: Integer version value
"""
d = defer.Deferred()
if self._check_connected(d):
return d
callback = self._zk_thread_callback(self._cb_set, d, path, data)
result = zookeeper.aset(self.handle, path, data, version, callback)
self._check_result(result, d, path=path)
return d
def _cb_set(self, d, path, data, result_code, node_stat):
if self._check_result(result_code, d, path=path):
return
d.callback(node_stat)
def set_connection_watcher(self, watcher):
"""
Sets a permanent global watcher on the connection. This will get
notice of changes to the connection state.
@param: watcher function
"""
if not callable(watcher):
raise SyntaxError("Invalid Watcher %r" % (watcher))
watcher = self._wrap_watcher(watcher, None, None)
zookeeper.set_watcher(self.handle, watcher)
def set_session_callback(self, callback):
"""Set a callback to receive session events.
Session events are by default ignored. Interested applications
may choose to set a session event watcher on the connection
to receive session events. Session events are typically broadcast
by the libzookeeper library to all extant watchers, but the
twisted integration using deferreds is not capable of receiving
multiple values (session events and watch events), so this
client implementation instead provides for a user defined callback
to be invoked with them instead. The callback receives a single
parameter, the session event in the form of a ClientEvent instance.
Additional details on session events
------------------------------------
http://bit.ly/mQrOMY
http://bit.ly/irKpfn
"""
if not callable(callback):
raise TypeError("Invalid callback %r" % callback)
self._session_event_callback = callback
def set_connection_error_callback(self, callback):
"""Set a callback to receive connection error exceptions.
By default the error will be raised when the client API
call is made. Setting a connection level error handler allows
applications to centralize their handling of connection loss,
instead of having to guard every zk interaction.
The callback receives two parameters, the client instance
and the exception.
"""
if not callable(callback):
raise TypeError("Invalid callback %r" % callback)
if self._connection_error_callback is not None:
raise RuntimeError((
"Connection error handlers can't be changed %s" %
self._connection_error_callback))
self._connection_error_callback = callback
def set_deterministic_order(self, boolean):
"""
The zookeeper client will by default randomize the server hosts
it will connect to unless this is set to True.
This is a global setting across connections.
"""
zookeeper.deterministic_conn_order(bool(boolean))
def sync(self, path="/"):
"""Flushes the connected zookeeper server with the leader.
@param path: The root path to flush, all child nodes are also flushed.
"""
d = defer.Deferred()
if self._check_connected(d):
return d
def _cb_sync(result_code, path):
if self._check_result(result_code, d, path=path):
return
d.callback(path)
callback = self._zk_thread_callback(_cb_sync)
result = zookeeper.async(self.handle, path, callback)
self._check_result(result, d, path=path)
return d
txzookeeper-0.9.8/txzookeeper/utils.py 0000664 0001750 0001750 00000005500 12144707107 020427 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010-2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
import zookeeper
from twisted.internet.defer import inlineCallbacks, Deferred
@inlineCallbacks
def retry_change(client, path, change_function):
"""
A utility function to execute a node change function, repeatedly
in the face of transient errors. The node at 'path's content will
be changed by the 'change_function' which is passed the node's
current content and node stat. The output will be used to replace
the node's content. If the new content is identical to the previous
new content, no changes are made. Automatically performs, retries
in the face of errors.
@param client A connected txzookeeper client
@param path A path to a node that will be modified
@param change_function A python function that will receive two parameters
the node_content and the current node stat, and will return the
new node content. The function must not have side-effects as
it will be called again in the event of various error conditions.
"""
while True:
create_mode = False
try:
content, stat = yield client.get(path)
except zookeeper.NoNodeException:
create_mode = True
content, stat = None, None
new_content = yield change_function(content, stat)
if new_content == content:
break
try:
if create_mode:
yield client.create(path, new_content)
else:
yield client.set(path, new_content, version=stat["version"])
break
except (zookeeper.NodeExistsException,
zookeeper.NoNodeException,
zookeeper.BadVersionException):
pass
def sleep(delay):
"""Non-blocking sleep.
:param int delay: time in seconds to sleep.
:return: a Deferred that fires after the desired delay.
:rtype: :class:`twisted.internet.defer.Deferred`
"""
from twisted.internet import reactor
deferred = Deferred()
reactor.callLater(delay, deferred.callback, None)
return deferred
txzookeeper-0.9.8/setup.py 0000664 0001750 0001750 00000004022 12135533751 016050 0 ustar kapil kapil 0000000 0000000 #
# Copyright (C) 2010, 2011 Canonical Ltd. All Rights Reserved
#
# This file is part of txzookeeper.
#
# Authors:
# Kapil Thangavelu
#
# txzookeeper is free software: you can redistribute it and/or modify
# it under the terms of the GNU Lesser General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# txzookeeper is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with txzookeeper. If not, see .
#
# Please see the files COPYING and COPYING.LESSER for the text of the
# License.
#
import sys, re
from setuptools import find_packages, setup
long_description = """
Twisted API for Apache Zookeeper. Includes a distributed lock, and several
queue implementations.
"""
# Parse directly to avoid build-time dependencies on zookeeper, twisted, etc.
for line in open("txzookeeper/__init__.py"):
m = re.match('version = "(.*)"', line)
if m:
version = m.group(1)
break
else:
sys.exit("error: can't find version information")
setup(
name="txzookeeper",
version=version,
description="Twisted api for Apache Zookeeper",
author="Juju Developers",
author_email="juju@lists.ubuntu.com",
url="https://launchpad.net/txzookeeper",
license="LGPL",
packages=find_packages(),
test_suite="txzookeeper.tests.egg_test_runner",
long_description=long_description,
classifiers=[
"Intended Audience :: Developers",
"Intended Audience :: System Administrators",
"Intended Audience :: Information Technology",
"Programming Language :: Python",
"Topic :: Database",
"License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)"
],
)
txzookeeper-0.9.8/setup.cfg 0000664 0001750 0001750 00000000073 12144725212 016154 0 ustar kapil kapil 0000000 0000000 [egg_info]
tag_build =
tag_date = 0
tag_svn_revision = 0
txzookeeper-0.9.8/PKG-INFO 0000664 0001750 0001750 00000001262 12144725212 015431 0 ustar kapil kapil 0000000 0000000 Metadata-Version: 1.1
Name: txzookeeper
Version: 0.9.8
Summary: Twisted api for Apache Zookeeper
Home-page: https://launchpad.net/txzookeeper
Author: Juju Developers
Author-email: juju@lists.ubuntu.com
License: LGPL
Description:
Twisted API for Apache Zookeeper. Includes a distributed lock, and several
queue implementations.
Platform: UNKNOWN
Classifier: Intended Audience :: Developers
Classifier: Intended Audience :: System Administrators
Classifier: Intended Audience :: Information Technology
Classifier: Programming Language :: Python
Classifier: Topic :: Database
Classifier: License :: OSI Approved :: GNU Library or Lesser General Public License (LGPL)