pax_global_header00006660000000000000000000000064150640121000014477gustar00rootroot0000000000000052 comment=faf54de652991899768d7075feab6fcc10b72ba7 danielfm-pybreaker-7c96106/000077500000000000000000000000001506401210000155215ustar00rootroot00000000000000danielfm-pybreaker-7c96106/.github/000077500000000000000000000000001506401210000170615ustar00rootroot00000000000000danielfm-pybreaker-7c96106/.github/FUNDING.yml000066400000000000000000000000451506401210000206750ustar00rootroot00000000000000github: danielfm liberapay: danielfm danielfm-pybreaker-7c96106/.github/workflows/000077500000000000000000000000001506401210000211165ustar00rootroot00000000000000danielfm-pybreaker-7c96106/.github/workflows/codeql-analysis.yml000066400000000000000000000044411506401210000247340ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ master ] pull_request: # The branches below must be a subset of the branches above branches: [ master ] schedule: - cron: '28 9 * * 4' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ] # Learn more about CodeQL language support at https://git.io/codeql-language-support steps: - name: Checkout repository uses: actions/checkout@v2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v1 # ℹī¸ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏ī¸ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 danielfm-pybreaker-7c96106/.github/workflows/release.yml000066400000000000000000000027541506401210000232710ustar00rootroot00000000000000name: Python Package Release on: push: branches: - main pull_request: branches: - main release: types: - published jobs: build: name: Build Python ${{ matrix.python-version }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: - uses: actions/checkout@v4 - name: Cache pyprojectx uses: actions/cache@v4 with: key: ${{ hashFiles('pyproject.toml') }}-${{ matrix.python-version }}-venvs path: | .pyprojectx venv - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Test and build run: | ./pw lint ./pw type-check ./pw test publish: name: Publish to PyPI runs-on: ubuntu-latest needs: build environment: pypi permissions: id-token: write if: github.event_name == 'release' && github.event.action == 'published' steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: "3.10" - name: Install build dependencies run: python -m pip install --upgrade pip build - name: Build package run: python -m build - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@release/v1 danielfm-pybreaker-7c96106/.gitignore000066400000000000000000000003451506401210000175130ustar00rootroot00000000000000# Compiled python code *.pyc # tox .tox .python-version # Build directories build/ dist/ *.egg-info/ .eggs/ .pyprojectx/ # Pycharm directories .idea .ropeproject/ *.iml # Visual Studio Code /.vscode/ # VirtualEnv /venv/ *~ danielfm-pybreaker-7c96106/.pre-commit-config.yaml000066400000000000000000000010211506401210000217740ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: fix-byte-order-marker - repo: local hooks: - id: format name: format entry: pw format language: script types: [ python ] pass_filenames: false - repo: local hooks: - id: lint name: lint entry: pw lint language: script types: [ python ] pass_filenames: false danielfm-pybreaker-7c96106/CHANGELOG000066400000000000000000000070061506401210000167360ustar00rootroot00000000000000Changelog ========= Version 1.4.0 (July 6, 2025) * Add success_threshold parameter (Thanks @sideshowbandana) Version 1.3.0 (March 5, 2025) * Dropped support for Python 3.8 * Fix 'DeprecationWarning: datetime.datetime.utcnow()' Version 1.2.0 (February 14, 2024) * Fixed type hints * Improved building and testing * Dropped support for Python 3.7 Version 1.1.0 (January 09, 2024) * Added calling() method to CircuitBreaker, returning a context manager (Thanks @martijnthe) Version 1.0.3 (December 22, 2023) * Added py.typed file (Thanks @martijnthe) Version 1.0.2 (June 9, 2023) * Update redis (Thanks @rodrigobraga) Version 1.0.1 (August 12, 2022) * Fix state parameter type hint (Thanks @Kludex) Version 1.0.0 (August 11, 2022) * Highlight Python code in README (Thanks @Kludex) * Add linter & format setup (Thanks @Kludex) * Drop Python 3.6 support (Thanks @Kludex) * Fully type annotate pybreaker.py (Thanks @Kludex) Version 0.8.0 (December 14, 2021) * Add support to redis cluster storage, without ttransaction on opened... (Thanks @felipeagger) * Replace Travis by GitHub Actions workflow Version 0.7.0 (March 3, 2021) * Option for throwing original error on circuit trip (Thanks @Cruuncher) Version 0.6.0 (October 22, 2019) * Use generator.throw to fix generator error raising (Thanks @phillbaker) Version 0.5.0 (March 19, 2019) * Add support for Python 3.7, where Stopiteration behaviour changes due to PEP 479 (Thanks @jshiell) * Allow excluded exceptions to be specified as callables (Thanks @jshiell) Version 0.4.5 (October 26, 2018) * Fix bug with timeout window growing with additional breakers defined (Thanks @shawndrape) Version 0.4.4 (May 21, 2018) * Fix PyPI release Version 0.4.3 (May 21, 2018) * Re-initialize state on Redis if missing (Thanks @scascketta) * Add trigger exception into the CircuitBreakerError (Thanks @tczhaodachuan) Version 0.4.2 (November 9, 2017) * Add optional name to CircuitBreaker (Thanks @chrisvaughn) Version 0.4.1 (October 2, 2017) * Get initial CircuitBreaker state from state_storage (Thanks @alukach) Version 0.4.0 (June 23, 2017) * Added optional support for asynchronous Tornado calls (Thanks @jeffrand) * Fixed typo (issue #19) (Thanks @jeffrand) Version 0.3.3 (June 2, 2017) * Fixed bug that caused pybreaker to break (!) if redis package was not present (Thanks @phillbaker) Version 0.3.2 (June 1, 2017) * Added support for optional Redis backing (Thanks @phillbaker) * Fixed: Should listener.failure be called when the circuit is closed and a call fails? (Thanks @sj175 for the report) * Fixed: Wrapped function is called twice during successful call in open state (Thanks @jstordeur) Version 0.3.1 (January 25, 2017) * Added support for optional Redis backing Version 0.3.0 (September 1, 2016) * Fixed generator issue. (Thanks @dpdornseifer) Version 0.2.3 (July 25, 2014) * Added support to generator functions. (Thanks @mauriciosl) Version 0.2.1 (October 23, 2010) * Fixed a few concurrency bugs. Version 0.2 (October 20, 2010) * Several API changes, breaks backwards compatibility. * New CircuitBreakerListener class that allows the user to listen to events in a circuit breaker without the need to subclass CircuitBreaker. * Decorator now uses 'functools.wraps' to avoid loss of information on decorated functions. Version 0.1.1 (October 17, 2010) * Instance of CircuitBreaker can now be used as a decorator. * Python 2.6+ is now required in order to make the same code base compatible with Python 3+. Version 0.1 (October 16, 2010) * First public release. danielfm-pybreaker-7c96106/LICENSE000066400000000000000000000030161506401210000165260ustar00rootroot00000000000000Copyright (c) 2010-2021, Daniel Fernandes Martins All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. Neither the name of PyBreaker nor the names of its contributors may be used to endorse or promote products derived from this software software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. danielfm-pybreaker-7c96106/README.rst000066400000000000000000000234611506401210000172160ustar00rootroot00000000000000 PyBreaker ========= PyBreaker is a Python implementation of the Circuit Breaker pattern, described in Michael T. Nygard's book `Release It!`_. In Nygard's words, *"circuit breakers exists to allow one subsystem to fail without destroying the entire system. This is done by wrapping dangerous operations (typically integration points) with a component that can circumvent calls when the system is not healthy"*. Features -------- * Configurable list of excluded exceptions (e.g. business exceptions) * Configurable failure threshold and reset timeout * Support for several event listeners per circuit breaker * Can guard generator functions * Functions and properties for easy monitoring and management * Thread-safe * Optional redis backing * Optional support for asynchronous Tornado calls Requirements ------------ * `Python`_ 3.9+ Installation ------------ Run the following command line to download the latest stable version of PyBreaker from `PyPI`_:: $ pip install pybreaker If you are a `Git`_ user, you might want to install the current development version in editable mode:: $ git clone git://github.com/danielfm/pybreaker.git $ cd pybreaker $ # run tests (on windows omit ./) $ ./pw test $ pip install -e . Usage ----- The first step is to create an instance of ``CircuitBreaker`` for each integration point you want to protect against: .. code:: python import pybreaker # Used in database integration points db_breaker = pybreaker.CircuitBreaker(fail_max=5, reset_timeout=60) ``CircuitBreaker`` instances should live globally inside the application scope, e.g., live across requests. You can also configure a success threshold to require multiple successful requests before closing the circuit breaker: .. code:: python # Require 3 successful requests before closing db_breaker = pybreaker.CircuitBreaker(fail_max=5, reset_timeout=60, success_threshold=3) .. note:: Integration points to external services (i.e. databases, queues, etc) are more likely to fail, so make sure to always use timeouts when accessing such services if there's support at the API level. If you'd like to use the Redis backing, initialize the ``CircuitBreaker`` with a ``CircuitRedisStorage``: .. code:: python import pybreaker import redis redis = redis.StrictRedis() db_breaker = pybreaker.CircuitBreaker( fail_max=5, reset_timeout=60, state_storage=pybreaker.CircuitRedisStorage(pybreaker.STATE_CLOSED, redis)) **Do not** initialize the Redis connection with the ``decode_responses`` set to ``True``, this will force returning ASCII objects from redis and in Python3+ will fail with: `AttributeError: 'str' object has no attribute 'decode'` .. note:: You may want to reuse a connection already created in your application, if you're using ``django_redis`` for example: .. code:: python import pybreaker from django_redis import get_redis_connection db_breaker = pybreaker.CircuitBreaker( fail_max=5, reset_timeout=60, state_storage=pybreaker.CircuitRedisStorage(pybreaker.STATE_CLOSED, get_redis_connection('default'))) .. note:: If you require multiple, independent CircuitBreakers and wish to store their states in Redis, it is essential to assign a ``unique namespace`` for each CircuitBreaker instance. This can be achieved by specifying a distinct namespace parameter in the CircuitRedisStorage constructor. for example: .. code:: python import pybreaker from django_redis import get_redis_connection db_breaker = pybreaker.CircuitBreaker( fail_max=5, reset_timeout=60, state_storage=pybreaker.CircuitRedisStorage(pybreaker.STATE_CLOSED, get_redis_connection('default'),namespace='unique_namespace')) Event Listening ``````````````` There's no need to subclass ``CircuitBreaker`` if you just want to take action when certain events occur. In that case, it's better to subclass ``CircuitBreakerListener`` instead: .. code:: python class DBListener(pybreaker.CircuitBreakerListener): "Listener used by circuit breakers that execute database operations." def before_call(self, cb, func, *args, **kwargs): "Called before the circuit breaker `cb` calls `func`." pass def state_change(self, cb, old_state, new_state): "Called when the circuit breaker `cb` state changes." pass def failure(self, cb, exc): "Called when a function invocation raises a system error." pass def success(self, cb): "Called when a function invocation succeeds." pass class LogListener(pybreaker.CircuitBreakerListener): "Listener used to log circuit breaker events." def state_change(self, cb, old_state, new_state): msg = "State Change: CB: {0}, New State: {1}".format(cb.name, new_state) logging.info(msg) To add listeners to a circuit breaker: .. code:: python # At creation time... db_breaker = pybreaker.CircuitBreaker(listeners=[DBListener(), LogListener()]) # ...or later db_breaker.add_listeners(OneListener(), AnotherListener()) What Does a Circuit Breaker Do? ``````````````````````````````` Let's say you want to use a circuit breaker on a function that updates a row in the ``customer`` database table: .. code:: python @db_breaker def update_customer(cust): # Do stuff here... pass # Will trigger the circuit breaker updated_customer = update_customer(my_customer) Or if you don't want to use the decorator syntax: .. code:: python def update_customer(cust): # Do stuff here... pass # Will trigger the circuit breaker updated_customer = db_breaker.call(update_customer, my_customer) Or use it as a context manager and a `with` statement: .. code:: python # Will trigger the circuit breaker with db_breaker.calling(): # Do stuff here... pass According to the default parameters, the circuit breaker ``db_breaker`` will automatically open the circuit after 5 consecutive failures in ``update_customer``. When the circuit is open, all calls to ``update_customer`` will fail immediately (raising ``CircuitBreakerError``) without any attempt to execute the real operation. If you want the original error to be thrown when the circuit trips, set the ``throw_new_error_on_trip`` option to ``False``: .. code:: python pybreaker.CircuitBreaker(..., throw_new_error_on_trip=False) After 60 seconds, the circuit breaker will allow the next call to ``update_customer`` pass through. If that call succeeds, the circuit is closed; if it fails, however, the circuit is opened again until another timeout elapses. Optional Tornado Support ```````````````````````` A circuit breaker can (optionally) be used to call asynchronous Tornado functions: .. code:: python from tornado import gen @db_breaker(__pybreaker_call_async=True) @gen.coroutine def async_update(cust): # Do async stuff here... pass Or if you don't want to use the decorator syntax: .. code:: python @gen.coroutine def async_update(cust): # Do async stuff here... pass updated_customer = db_breaker.call_async(async_update, my_customer) Excluding Exceptions ```````````````````` By default, a failed call is any call that raises an exception. However, it's common to raise exceptions to also indicate business exceptions, and those exceptions should be ignored by the circuit breaker as they don't indicate system errors: .. code:: python # At creation time... db_breaker = CircuitBreaker(exclude=[CustomerValidationError]) # ...or later db_breaker.add_excluded_exception(CustomerValidationError) In that case, when any function guarded by that circuit breaker raises ``CustomerValidationError`` (or any exception derived from ``CustomerValidationError``), that call won't be considered a system failure. So as to cover cases where the exception class alone is not enough to determine whether it represents a system error, you may also pass a callable rather than a type: .. code:: python db_breaker = CircuitBreaker(exclude=[lambda e: type(e) == HTTPError and e.status_code < 500]) You may mix types and filter callables freely. Monitoring and Management ````````````````````````` A circuit breaker provides properties and functions you can use to monitor and change its current state: .. code:: python # Get the current number of consecutive failures print(db_breaker.fail_counter) # Get the current number of consecutive successes print(db_breaker.success_counter) # Get/set the maximum number of consecutive failures print(db_breaker.fail_max) db_breaker.fail_max = 10 # Get/set the success threshold print(db_breaker.success_threshold) db_breaker.success_threshold = 3 # Get/set the current reset timeout period (in seconds) print db_breaker.reset_timeout db_breaker.reset_timeout = 60 # Get the current state, i.e., 'open', 'half-open', 'closed' print(db_breaker.current_state) # Closes the circuit db_breaker.close() # Half-opens the circuit db_breaker.half_open() # Opens the circuit db_breaker.open() These properties and functions might and should be exposed to the operations staff somehow as they help them to detect problems in the system. Contributing ------------- Run tests:: $ ./pw test Code formatting (black and isort) and linting (mypy) :: $ ./pw format $ ./pw lint Above commands will automatically install the necessary tools inside *.pyprojectx* and also install pre-commit hooks. List available commands:: $ ./pw -i .. _Python: http://python.org .. _Jython: http://jython.org .. _Release It!: https://pragprog.com/titles/mnee2/release-it-second-edition/ .. _PyPI: http://pypi.python.org .. _Git: http://git-scm.com danielfm-pybreaker-7c96106/pw000077500000000000000000000156431506401210000161060ustar00rootroot00000000000000#!/usr/bin/env python3 ################################################################################## # Pyprojectx wrapper script # # https://github.com/pyprojectx/pyprojectx # # # # Copyright (c) 2021 Ivo Houbrechts # # # # Licensed under the MIT license # ################################################################################## import argparse import os import subprocess import sys from pathlib import Path from venv import EnvBuilder VERSION = "2.0.8" PYPROJECTX_INSTALL_DIR_ENV_VAR = "PYPROJECTX_INSTALL_DIR" PYPROJECTX_PACKAGE_ENV_VAR = "PYPROJECTX_PACKAGE" PYPROJECT_TOML = "pyproject.toml" DEFAULT_INSTALL_DIR = ".pyprojectx" CYAN = "\033[96m" BLUE = "\033[94m" RED = "\033[91m" RESET = "\033[0m" if sys.platform.startswith("win"): os.system("color") def run(args): try: options = get_options(args) pyprojectx_script = ensure_pyprojectx(options) explicit_options = [] if not options.toml: explicit_options += ["--toml", str(options.toml_path)] if not options.install_dir: explicit_options += ["--install-dir", str(options.install_path)] subprocess.run([str(pyprojectx_script), *explicit_options, *args], check=True) except subprocess.CalledProcessError as e: raise SystemExit(e.returncode) from e def get_options(args): options = arg_parser().parse_args(args) options.install_path = Path( options.install_dir or os.environ.get( PYPROJECTX_INSTALL_DIR_ENV_VAR, Path(__file__).with_name(DEFAULT_INSTALL_DIR), ) ) options.toml_path = ( Path(options.toml) if options.toml else Path(__file__).with_name(PYPROJECT_TOML) ) if os.environ.get(PYPROJECTX_PACKAGE_ENV_VAR): options.version = "development" options.pyprojectx_package = os.environ.get(PYPROJECTX_PACKAGE_ENV_VAR) else: options.version = VERSION options.pyprojectx_package = f"pyprojectx~={VERSION}" options.verbosity = ( 0 if options.quiet or not options.verbosity else options.verbosity ) return options def arg_parser(): parser = argparse.ArgumentParser( description="Execute commands or aliases defined in the [tool.pyprojectx] section of pyproject.toml. " "Use the -i or --info option to see available tools and aliases.", allow_abbrev=False, ) parser.add_argument("--version", action="version", version=VERSION) parser.add_argument( "--toml", "-t", action="store", help="The toml config file. Defaults to 'pyproject.toml' in the same directory as the pw script.", ) parser.add_argument( "--install-dir", action="store", help=f"The directory where all tools (including pyprojectx) are installed; defaults to the " f"{PYPROJECTX_INSTALL_DIR_ENV_VAR} environment value if set, else '.pyprojectx' " f"in the same directory as the invoked pw script", ) parser.add_argument( "--force-install", "-f", action="store_true", help="Force clean installation of the virtual environment used to run cmd, if any", ) parser.add_argument( "--install-context", action="store", metavar="tool-context", help="Install a tool context without actually running any command.", ) parser.add_argument( "--verbose", "-v", action="count", dest="verbosity", help="Give more output. This option is additive and can be used up to 2 times.", ) parser.add_argument( "--quiet", "-q", action="store_true", help="Suppress output", ) parser.add_argument( "--info", "-i", action="store_true", help="Show the configuration details of a command instead of running it. " "If no command is specified, a list with all available tools and aliases is shown.", ) parser.add_argument( "--add", action="store", metavar="[context:],...", help="Add one or more packages to a tool context. " "If no context is specified, the packages are added to the main context. " "Packages can be specified as in 'pip install', except that a ',' can't be used in the version specification.", ) parser.add_argument( "--lock", action="store_true", help="Write all dependencies of all tool contexts to 'pw.lock' to guarantee reproducible outcomes.", ) parser.add_argument( "--install-px", action="store_true", help="Install the px and pxg scripts in your home directory.", ) parser.add_argument( "--upgrade", action="store_true", help="Print instructions to download the latest pyprojectx wrapper scripts.", ) parser.add_argument( "command", nargs=argparse.REMAINDER, help="The command/alias with optional arguments to execute.", ) return parser def ensure_pyprojectx(options): env_builder = EnvBuilder(with_pip=True) venv_dir = options.install_path.joinpath( "pyprojectx", f"{options.version}-py{sys.version_info.major}.{sys.version_info.minor}", ) env_context = env_builder.ensure_directories(venv_dir) pyprojectx_script = Path(env_context.bin_path, "pyprojectx") pyprojectx_exe = Path(env_context.bin_path, "pyprojectx.exe") pip_cmd = [env_context.env_exe, "-m", "pip", "install"] if options.quiet: out = subprocess.DEVNULL pip_cmd.append("--quiet") else: out = sys.stderr if not pyprojectx_script.is_file() and not pyprojectx_exe.is_file(): if not options.quiet: print( f"{CYAN}creating pyprojectx venv in {BLUE}{venv_dir}{RESET}", file=sys.stderr, ) env_builder.create(venv_dir) subprocess.run( [*pip_cmd, "--upgrade", "pip"], stdout=out, check=True, ) if not options.quiet: print( f"{CYAN}installing pyprojectx {BLUE}{options.version}: {options.pyprojectx_package} {RESET}", file=sys.stderr, ) if options.version == "development": if not options.quiet: print( f"{RED}WARNING: {options.pyprojectx_package} is installed in editable mode{RESET}", file=sys.stderr, ) pip_cmd.append("-e") subprocess.run([*pip_cmd, options.pyprojectx_package], stdout=out, check=True) return pyprojectx_script if __name__ == "__main__": run(sys.argv[1:]) danielfm-pybreaker-7c96106/pw.bat000066400000000000000000000000361506401210000166360ustar00rootroot00000000000000@echo off python "%~dp0pw" %* danielfm-pybreaker-7c96106/pw.lock000066400000000000000000000010711506401210000170200ustar00rootroot00000000000000[main] requirements = ["PyYAML==6.0.1", "certifi==2024.2.2", "cfgv==3.4.0", "charset-normalizer==3.3.2", "distlib==0.3.8", "docutils==0.20.1", "filelock==3.13.1", "flit==3.9.0", "flit_core==3.9.0", "identify==2.5.33", "idna==3.6", "mypy-extensions==1.0.0", "mypy==1.8.0", "nodeenv==1.8.0", "platformdirs==4.2.0", "pre-commit==3.5.0", "px-utils==1.1.0", "requests==2.31.0", "ruff==0.2.0", "tomli==2.0.1", "tomli_w==1.0.0", "typing_extensions==4.9.0", "urllib3==2.2.0", "virtualenv==20.25.0"] hash = "c05140614e209ba7f91f796ccdc08e52" post-install = "pre-commit install" danielfm-pybreaker-7c96106/pyproject.toml000066400000000000000000000043551506401210000204440ustar00rootroot00000000000000[project] name = "pybreaker" version = "1.4.1" description = "Python implementation of the Circuit Breaker pattern" authors = [ { name = "Daniel Fernandes Martins", email = "daniel.tritone@gmail.com" }, ] readme = "README.rst" urls = { Source = "http://github.com/danielfm/pybreaker" } license = { file = "LICENSE" } keywords = ["design", "pattern", "circuit", "breaker", "integration"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "License :: OSI Approved :: BSD License", "Programming Language :: Python", "Programming Language :: Python :: 3", "Topic :: Software Development :: Libraries", ] requires-python = ">=3.9" [project.optional-dependencies] test = ["pytest", "mock", "tornado", "redis", "fakeredis", "types-mock", "types-redis"] [build-system] requires = ["flit_core >=3.9"] build-backend = "flit_core.buildapi" [tool.flit.sdist] exclude = [".github", "tests", ".gitignore", ".pre-commit-config.yaml", "pw*"] [tool.pyprojectx] [tool.pyprojectx.main] requirements = ["pre-commit<=3.6", "ruff", "mypy", "px-utils", "flit"] post-install = "pre-commit install" [tool.pyprojectx.venv] dir = "@PROJECT_DIR/venv" requirements = ["pytest", "mock", "tornado", "redis", "fakeredis", "types-mock", "types-redis", "-e ."] [tool.pyprojectx.aliases] install = "pw@ --install-context venv" format = ["ruff format", "ruff check --select I --fix"] lint = ["@install", "ruff check src"] type-check = ["@install", "mypy --python-executable venv/bin/python --no-incremental"] fix-ruff = "ruff check --fix" test = { cmd = "pytest", ctx = "venv" } clean = "pxrm .eggs venv src/pybreaker.egg-info" [tool.mypy] disallow_untyped_defs = true ignore_missing_imports = true no_implicit_optional = true show_error_codes = true files = ["src", "tests/typechecks.py"] [tool.ruff] line-length = 120 target-version = "py39" [tool.ruff.lint] select = ["ALL"] fixable = ["ALL"] ignore = ["ISC001", "ANN", "FA", "FBT", "D100", "D102", "D205", "D103", "D104", "D105", "D213", "D203", "T201", "TRY003", "EM102", "COM812", "S602", "S603", "S604", "S605", "S607", "S324"] [tool.ruff.lint.per-file-ignores] "src/*" = ["PLR0913", "TCH002", "D401", "D404", "SLF001", "BLE001", "TRY301", "DTZ001", "DTZ003", "SIM102", "TRY300"] danielfm-pybreaker-7c96106/src/000077500000000000000000000000001506401210000163105ustar00rootroot00000000000000danielfm-pybreaker-7c96106/src/pybreaker/000077500000000000000000000000001506401210000202745ustar00rootroot00000000000000danielfm-pybreaker-7c96106/src/pybreaker/__init__.py000066400000000000000000001014201506401210000224030ustar00rootroot00000000000000"""Threadsafe pure-Python implementation of the Circuit Breaker pattern, described by Michael T. Nygard in his book 'Release It!'. For more information on this and other patterns and best practices, buy the book at https://pragprog.com/titles/mnee2/release-it-second-edition/ """ from __future__ import annotations import calendar import contextlib import logging import sys import threading import time import types from abc import abstractmethod from datetime import datetime, timedelta from functools import wraps from typing import ( TYPE_CHECKING, Any, Callable, Literal, NoReturn, TypeVar, Union, cast, overload, ) if TYPE_CHECKING: from collections.abc import Generator, Iterable, Sequence # For compatibility with Python 3.10 and earlier. # Otherwise, `from datetime import UTC` would suffice. try: from datetime import UTC # type: ignore[attr-defined] except ImportError: from datetime import timezone UTC = timezone.utc try: from tornado import gen HAS_TORNADO_SUPPORT = True except ImportError: HAS_TORNADO_SUPPORT = False try: from redis import Redis from redis.client import Pipeline from redis.exceptions import RedisError HAS_REDIS_SUPPORT = True except ImportError: HAS_REDIS_SUPPORT = False __all__ = ( "CircuitBreaker", "CircuitBreakerListener", "CircuitBreakerError", "CircuitMemoryStorage", "CircuitRedisStorage", "STATE_OPEN", "STATE_CLOSED", "STATE_HALF_OPEN", ) STATE_OPEN = "open" STATE_CLOSED = "closed" STATE_HALF_OPEN = "half-open" T = TypeVar("T") ExceptionType = TypeVar("ExceptionType", bound=BaseException) CBListenerType = TypeVar("CBListenerType", bound="CircuitBreakerListener") CBStateType = Union["CircuitClosedState", "CircuitHalfOpenState", "CircuitOpenState"] class CircuitBreaker: """More abstractly, circuit breakers exists to allow one subsystem to fail without destroying the entire system. This is done by wrapping dangerous operations (typically integration points) with a component that can circumvent calls when the system is not healthy. This pattern is described by Michael T. Nygard in his book 'Release It!'. """ def __init__( self, fail_max: int = 5, reset_timeout: float = 60, success_threshold: int = 1, exclude: Iterable[type[ExceptionType] | Callable[[Any], bool]] | None = None, listeners: Sequence[CBListenerType] | None = None, state_storage: CircuitBreakerStorage | None = None, name: str | None = None, throw_new_error_on_trip: bool = True, ) -> None: """Create a new circuit breaker with the given parameters.""" self._lock = threading.RLock() self._state_storage = state_storage or CircuitMemoryStorage(STATE_CLOSED) self._state = self._create_new_state(self.current_state) self._fail_max = fail_max self._reset_timeout = reset_timeout self._success_threshold = success_threshold self._excluded_exceptions = list(exclude or []) self._listeners = list(listeners or []) self._name = name self._throw_new_error_on_trip = throw_new_error_on_trip @property def fail_counter(self) -> int: """Return the current number of consecutive failures.""" return self._state_storage.counter @property def success_counter(self) -> int: """Return the current number of consecutive successes in half-open state.""" return self._state_storage.success_counter @property def fail_max(self) -> int: """Return the maximum number of failures tolerated before the circuit is opened.""" return self._fail_max @fail_max.setter def fail_max(self, number: int) -> None: """Set the maximum `number` of failures tolerated before the circuit is opened.""" self._fail_max = number @property def reset_timeout(self) -> float: """Once this circuit breaker is opened, it should remain opened until the timeout period, in seconds, elapses. """ return self._reset_timeout @reset_timeout.setter def reset_timeout(self, timeout: float) -> None: """Set the `timeout` period, in seconds, this circuit breaker should be kept open.""" self._reset_timeout = timeout @property def success_threshold(self) -> int: """Return the number of successful requests required before transitioning from half-open to closed state.""" return self._success_threshold @success_threshold.setter def success_threshold(self, threshold: int) -> None: """Set the number of successful requests required before transitioning from half-open to closed state.""" self._success_threshold = threshold def _create_new_state( self, new_state: str, prev_state: CircuitBreakerState | None = None, notify: bool = False, ) -> CBStateType: """Return state object from state string, i.e., 'closed' -> .""" state_map: dict[str, type[CBStateType]] = { STATE_CLOSED: CircuitClosedState, STATE_OPEN: CircuitOpenState, STATE_HALF_OPEN: CircuitHalfOpenState, } try: cls = state_map[new_state] return cls(self, prev_state=prev_state, notify=notify) except KeyError as e: msg = "Unknown state {!r}, valid states: {}" raise ValueError(msg.format(new_state, ", ".join(state_map))) from e @property def state(self) -> CBStateType: """Update (if needed) and returns the cached state object.""" # Ensure cached state is up-to-date if self.current_state != self._state.name: # If cached state is out-of-date, that means that it was likely # changed elsewhere (e.g. another process instance). We still send # out a notification, informing others that this particular circuit # breaker instance noticed the changed circuit. self.state = self.current_state # type: ignore[assignment] return self._state @state.setter def state(self, state_str: str) -> None: """Set cached state and notify listeners of newly cached state.""" with self._lock: self._state = self._create_new_state(state_str, prev_state=self._state, notify=True) @property def current_state(self) -> str: """Return a string that identifies the state of the circuit breaker as reported by the _state_storage. i.e., 'closed', 'open', 'half-open'. """ return self._state_storage.state @property def excluded_exceptions( self, ) -> tuple[type[ExceptionType] | Callable[[Any], bool], ...]: """Return the list of excluded exceptions, e.g., exceptions that should not be considered system errors by this circuit breaker. """ return tuple(self._excluded_exceptions) def add_excluded_exception(self, exception: type[ExceptionType]) -> None: """Add an exception to the list of excluded exceptions.""" with self._lock: self._excluded_exceptions.append(exception) def add_excluded_exceptions(self, *exceptions: type[ExceptionType]) -> None: """Add exceptions to the list of excluded exceptions.""" for exc in exceptions: self.add_excluded_exception(exc) def remove_excluded_exception(self, exception: type[ExceptionType]) -> None: """Remove an exception from the list of excluded exceptions.""" with self._lock: self._excluded_exceptions.remove(exception) def _inc_counter(self) -> None: """Increment the counter of failed calls.""" self._state_storage.increment_counter() def is_system_error(self, exception: ExceptionType) -> bool: """Return whether the exception `exception` is considered a signal of system malfunction. Business exceptions should not cause this circuit breaker to open. """ exception_type = type(exception) for exclusion in self._excluded_exceptions: if type(exclusion) is type: if issubclass(exception_type, exclusion): return False elif callable(exclusion): if exclusion(exception): return False return True def call(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T: """Call `func` with the given `args` and `kwargs` according to the rules implemented by the current state of this circuit breaker. """ with self._lock: return self.state.call(func, *args, **kwargs) @contextlib.contextmanager def calling(self) -> Any: """Return a context manager, enabling the circuit breaker to be used with a `with` statement. The block of code inside the `with` statement will be executed according to the rules implemented by the current state of this circuit breaker. """ def _wrapper() -> Generator: yield yield from self.call(_wrapper) def call_async(self, func, *args, **kwargs): # type: ignore[no-untyped-def] """Call async `func` with the given `args` and `kwargs` according to the rules implemented by the current state of this circuit breaker. Return a closure to prevent import errors when using without tornado present """ @gen.coroutine def wrapped(): # type: ignore[no-untyped-def] with self._lock: ret = yield self.state.call_async(func, *args, **kwargs) raise gen.Return(ret) return wrapped() def open(self) -> bool: """Open the circuit, e.g., the following calls will immediately fail until timeout elapses.""" with self._lock: self._state_storage.opened_at = datetime.now(UTC) self.state = self._state_storage.state = STATE_OPEN # type: ignore[assignment] return self._throw_new_error_on_trip def half_open(self) -> None: """Half-open the circuit, e.g. lets the following call pass through and opens the circuit if the call fails (or closes the circuit if the call succeeds). """ with self._lock: self.state = self._state_storage.state = STATE_HALF_OPEN # type: ignore[assignment] def close(self) -> None: """Close the circuit, e.g. lets the following calls execute as usual.""" with self._lock: self._state_storage.reset_success_counter() # Reset success counter when closing self.state = self._state_storage.state = STATE_CLOSED # type: ignore[assignment] def __call__(self, *call_args: Any, **call_kwargs: bool) -> Callable: """Return a wrapper that calls the function `func` according to the rules implemented by the current state of this circuit breaker. Optionally takes the keyword argument `__pybreaker_call_coroutine`, which will will call `func` as a Tornado co-routine. """ call_async = call_kwargs.pop("__pybreaker_call_async", False) if call_async and not HAS_TORNADO_SUPPORT: message = "No module named tornado" raise ImportError(message) def _outer_wrapper(func): # type: ignore[no-untyped-def] @wraps(func) def _inner_wrapper(*args, **kwargs): # type: ignore[no-untyped-def] if call_async: return self.call_async(func, *args, **kwargs) return self.call(func, *args, **kwargs) return _inner_wrapper if call_args: return _outer_wrapper(*call_args) return _outer_wrapper @property def listeners(self) -> tuple[CBListenerType, ...]: """Return the registered listeners as a tuple.""" return tuple(self._listeners) # type: ignore[arg-type] def add_listener(self, listener: CBListenerType) -> None: """Register a listener for this circuit breaker.""" with self._lock: self._listeners.append(listener) # type: ignore[arg-type] def add_listeners(self, *listeners: CBListenerType) -> None: """Register listeners for this circuit breaker.""" for listener in listeners: self.add_listener(listener) def remove_listener(self, listener: CBListenerType) -> None: """Unregister a listener of this circuit breaker.""" with self._lock: self._listeners.remove(listener) # type: ignore[arg-type] @property def name(self) -> str | None: """Return the name of this circuit breaker. Useful for logging.""" return self._name @name.setter def name(self, name: str) -> None: """Set the name of this circuit breaker.""" self._name = name class CircuitBreakerStorage: """Define the underlying storage for a circuit breaker - the underlying implementation should be in a subclass that overrides the method this class defines. """ def __init__(self, name: str) -> None: """Create a new instance identified by `name`.""" self._name = name @property def name(self) -> str: """Return a human friendly name that identifies this state.""" return self._name @property @abstractmethod def state(self) -> str: """Override this method to retrieve the current circuit breaker state.""" @state.setter def state(self, state: str) -> None: """Override this method to set the current circuit breaker state.""" def increment_counter(self) -> None: """Override this method to increase the failure counter by one.""" def reset_counter(self) -> None: """Override this method to set the failure counter to zero.""" def increment_success_counter(self) -> None: """Override this method to increase the success counter by one.""" def reset_success_counter(self) -> None: """Override this method to set the success counter to zero.""" @property @abstractmethod def counter(self) -> int: """Override this method to retrieve the current value of the failure counter.""" @property @abstractmethod def success_counter(self) -> int: """Override this method to retrieve the current value of the success counter.""" @property @abstractmethod def opened_at(self) -> datetime | None: """Override this method to retrieve the most recent value of when the circuit was opened.""" @opened_at.setter def opened_at(self, datetime: datetime) -> None: """Override this method to set the most recent value of when the circuit was opened.""" class CircuitMemoryStorage(CircuitBreakerStorage): """Implement a `CircuitBreakerStorage` in local memory.""" def __init__(self, state: str) -> None: """Create a new instance with the given `state`.""" super().__init__("memory") self._fail_counter = 0 self._success_counter = 0 self._opened_at: datetime | None = None self._state = state @property def state(self) -> str: """Return the current circuit breaker state.""" return self._state @state.setter def state(self, state: str) -> None: """Set the current circuit breaker state to `state`.""" self._state = state def increment_counter(self) -> None: """Increase the failure counter by one.""" self._fail_counter += 1 def reset_counter(self) -> None: """Set the failure counter to zero.""" self._fail_counter = 0 def increment_success_counter(self) -> None: """Increase the success counter by one.""" self._success_counter += 1 def reset_success_counter(self) -> None: """Set the success counter to zero.""" self._success_counter = 0 @property def counter(self) -> int: """Return the current value of the failure counter.""" return self._fail_counter @property def success_counter(self) -> int: """Return the current value of the success counter.""" return self._success_counter @property def opened_at(self) -> datetime | None: """Return the most recent value of when the circuit was opened.""" return self._opened_at @opened_at.setter def opened_at(self, datetime: datetime) -> None: """Set the most recent value of when the circuit was opened to `datetime`.""" self._opened_at = datetime class CircuitRedisStorage(CircuitBreakerStorage): """Implement a `CircuitBreakerStorage` using redis.""" BASE_NAMESPACE = "pybreaker" logger = logging.getLogger(__name__) def __init__( self, state: str, redis_object: Redis, namespace: str | None = None, fallback_circuit_state: str = STATE_CLOSED, cluster_mode: bool = False, ): """Create a new instance with the given `state` and `redis` object. The redis object should be similar to pyredis' StrictRedis class. If there are any connection issues with redis, the `fallback_circuit_state` is used to determine the state of the circuit. """ # Module does not exist, so this feature is not available if not HAS_REDIS_SUPPORT: message = "CircuitRedisStorage can only be used if the required dependencies exist" raise ImportError(message) super().__init__("redis") self._redis = redis_object self._namespace_name = namespace self._fallback_circuit_state = fallback_circuit_state self._initial_state = str(state) self._cluster_mode = cluster_mode self._initialize_redis_state(self._initial_state) def _initialize_redis_state(self, state: str) -> None: self._redis.setnx(self._namespace("fail_counter"), 0) self._redis.setnx(self._namespace("success_counter"), 0) self._redis.setnx(self._namespace("state"), state) @property def state(self) -> str: """Return the current circuit breaker state. If the circuit breaker state on Redis is missing, re-initialize it with the fallback circuit state and reset the fail counter. """ try: state_bytes: bytes | None = self._redis.get(self._namespace("state")) except RedisError: self.logger.exception("RedisError: falling back to default circuit state") return self._fallback_circuit_state state = self._fallback_circuit_state if state_bytes is not None: state = state_bytes.decode("utf-8") else: # state retrieved from redis was missing, so we re-initialize # the circuit breaker state on redis self._initialize_redis_state(self._fallback_circuit_state) return state @state.setter def state(self, state: str) -> None: """Set the current circuit breaker state to `state`.""" try: self._redis.set(self._namespace("state"), str(state)) except RedisError: self.logger.exception("RedisError") def increment_counter(self) -> None: """Increase the failure counter by one.""" try: self._redis.incr(self._namespace("fail_counter")) except RedisError: self.logger.exception("RedisError") def reset_counter(self) -> None: """Set the failure counter to zero.""" try: self._redis.set(self._namespace("fail_counter"), 0) except RedisError: self.logger.exception("RedisError") def increment_success_counter(self) -> None: """Increase the success counter by one.""" try: self._redis.incr(self._namespace("success_counter")) except RedisError: self.logger.exception("RedisError") def reset_success_counter(self) -> None: """Set the success counter to zero.""" try: self._redis.set(self._namespace("success_counter"), 0) except RedisError: self.logger.exception("RedisError") @property def counter(self) -> int: """Return the current value of the failure counter.""" try: value = self._redis.get(self._namespace("fail_counter")) if value: return int(value) return 0 except RedisError: self.logger.exception("RedisError: Assuming no errors") return 0 @property def success_counter(self) -> int: """Return the current value of the success counter.""" try: value = self._redis.get(self._namespace("success_counter")) if value: return int(value) return 0 except RedisError: self.logger.exception("RedisError: Assuming no successes") return 0 @property def opened_at(self) -> datetime | None: """Returns a datetime object of the most recent value of when the circuit was opened.""" try: timestamp = self._redis.get(self._namespace("opened_at")) if timestamp: return datetime(*time.gmtime(int(timestamp))[:6], tzinfo=UTC) except RedisError: self.logger.exception("RedisError") return None @opened_at.setter def opened_at(self, now: datetime) -> None: """Atomically set the most recent value of when the circuit was opened to `now`. Stored in redis as a simple integer of unix epoch time. To avoid timezone issues between different systems, the passed in datetime should be in UTC. """ try: key = self._namespace("opened_at") if self._cluster_mode: current_value = self._redis.get(key) next_value = int(calendar.timegm(now.timetuple())) if not current_value or next_value > int(current_value): self._redis.set(key, next_value) else: def set_if_greater(pipe: Pipeline[bytes]) -> None: current_value = cast(bytes, pipe.get(key)) next_value = int(calendar.timegm(now.timetuple())) pipe.multi() if not current_value or next_value > int(current_value): pipe.set(key, next_value) self._redis.transaction(set_if_greater, key) except RedisError: self.logger.exception("RedisError") def _namespace(self, key: str) -> str: name_parts = [self.BASE_NAMESPACE, key] if self._namespace_name: name_parts.insert(0, self._namespace_name) return ":".join(name_parts) class CircuitBreakerListener: """Listener class used to plug code to a ``CircuitBreaker`` instance when certain events happen.""" def before_call(self, cb: CircuitBreaker, func: Callable[..., T], *args: Any, **kwargs: Any) -> None: """This callback function is called before the circuit breaker `cb` calls `fn`.""" def failure(self, cb: CircuitBreaker, exc: BaseException) -> None: """This callback function is called when a function called by the circuit breaker `cb` fails.""" def success(self, cb: CircuitBreaker) -> None: """This callback function is called when a function called by the circuit breaker `cb` succeeds.""" def state_change( self, cb: CircuitBreaker, old_state: CircuitBreakerState | None, new_state: CircuitBreakerState, ) -> None: """This callback function is called when the state of the circuit breaker `cb` state changes.""" class CircuitBreakerState: """Implement the behavior needed by all circuit breaker states.""" def __init__(self, cb: CircuitBreaker, name: str) -> None: """Create a new instance associated with the circuit breaker `cb` and identified by `name`.""" self._breaker: CircuitBreaker = cb self._name: str = name @property def name(self) -> str: """Return a human friendly name that identifies this state.""" return self._name @overload def _handle_error(self, exc: BaseException, reraise: Literal[True] = ...) -> NoReturn: ... @overload def _handle_error(self, exc: BaseException, reraise: Literal[False] = ...) -> None: ... def _handle_error(self, exc: BaseException, reraise: bool = True) -> None: """Handle a failed call to the guarded operation.""" if self._breaker.is_system_error(exc): self._breaker._inc_counter() for listener in self._breaker.listeners: listener.failure(self._breaker, exc) self.on_failure(exc) else: self._handle_success() if reraise: raise exc def _handle_success(self) -> None: """Handle a successful call to the guarded operation.""" self._breaker._state_storage.reset_counter() self.on_success() for listener in self._breaker.listeners: listener.success(self._breaker) def call(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T: """Calls `func` with the given `args` and `kwargs`, and updates the circuit breaker state according to the result. """ ret = None self.before_call(func, *args, **kwargs) for listener in self._breaker.listeners: listener.before_call(self._breaker, func, *args, **kwargs) try: ret = func(*args, **kwargs) if isinstance(ret, types.GeneratorType): return self.generator_call(ret) except BaseException as e: self._handle_error(e) else: self._handle_success() return ret def call_async(self, func, *args: Any, **kwargs: Any): # type: ignore[no-untyped-def] """Call async `func` with the given `args` and `kwargs`, and updates the circuit breaker state according to the result. Return a closure to prevent import errors when using without tornado present """ @gen.coroutine def wrapped(): # type: ignore[no-untyped-def] ret = None self.before_call(func, *args, **kwargs) for listener in self._breaker.listeners: listener.before_call(self._breaker, func, *args, **kwargs) try: ret = yield func(*args, **kwargs) if isinstance(ret, types.GeneratorType): raise gen.Return(self.generator_call(ret)) except BaseException as e: self._handle_error(e) else: self._handle_success() raise gen.Return(ret) return wrapped() def generator_call(self, wrapped_generator): # type: ignore[no-untyped-def] try: value = yield next(wrapped_generator) while True: value = yield wrapped_generator.send(value) except StopIteration: self._handle_success() return except BaseException as e: self._handle_error(e, reraise=False) wrapped_generator.throw(e) def before_call(self, func: Callable[..., Any], *args: Any, **kwargs: Any) -> None: """Override this method to be notified before a call to the guarded operation is attempted.""" def on_success(self) -> None: """Override this method to be notified when a call to the guarded operation succeeds.""" def on_failure(self, exc: BaseException) -> None: """Override this method to be notified when a call to the guarded operation fails.""" class CircuitClosedState(CircuitBreakerState): """In the normal "closed" state, the circuit breaker executes operations as usual. If the call succeeds, nothing happens. If it fails, however, the circuit breaker makes a note of the failure. Once the number of failures exceeds a threshold, the circuit breaker trips and "opens" the circuit. """ def __init__( self, cb: CircuitBreaker, prev_state: CircuitBreakerState | None = None, notify: bool = False, ) -> None: """Move the given circuit breaker `cb` to the "closed" state.""" super().__init__(cb, STATE_CLOSED) if notify: # We only reset the counter if notify is True, otherwise the CircuitBreaker # will lose it's failure count due to a second CircuitBreaker being created # using the same _state_storage object, or if the _state_storage objects # share a central source of truth (as would be the case with the redis # storage). self._breaker._state_storage.reset_counter() for listener in self._breaker.listeners: listener.state_change(self._breaker, prev_state, self) def on_failure(self, exc: BaseException) -> None: """Move the circuit breaker to the "open" state once the failures threshold is reached.""" if self._breaker._state_storage.counter >= self._breaker.fail_max: throw_new_error = self._breaker.open() if throw_new_error: error_msg = "Failures threshold reached, circuit breaker opened" raise CircuitBreakerError(error_msg).with_traceback(sys.exc_info()[2]) raise exc class CircuitOpenState(CircuitBreakerState): """When the circuit is "open", calls to the circuit breaker fail immediately, without any attempt to execute the real operation. This is indicated by the ``CircuitBreakerError`` exception. After a suitable amount of time, the circuit breaker decides that the operation has a chance of succeeding, so it goes into the "half-open" state. """ def __init__( self, cb: CircuitBreaker, prev_state: CircuitBreakerState | None = None, notify: bool = False, ) -> None: """Move the given circuit breaker `cb` to the "open" state.""" super().__init__(cb, STATE_OPEN) if notify: # Reset success counter when opening the circuit self._breaker._state_storage.reset_success_counter() for listener in self._breaker.listeners: listener.state_change(self._breaker, prev_state, self) def before_call(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T: """After the timeout elapses, move the circuit breaker to the "half-open" state; otherwise, raises ``CircuitBreakerError`` without any attempt to execute the real operation. """ timeout = timedelta(seconds=self._breaker.reset_timeout) opened_at = self._breaker._state_storage.opened_at if opened_at and datetime.now(UTC) < opened_at + timeout: error_msg = "Timeout not elapsed yet, circuit breaker still open" raise CircuitBreakerError(error_msg) self._breaker.half_open() return self._breaker.call(func, *args, **kwargs) def call(self, func: Callable[..., T], *args: Any, **kwargs: Any) -> T: """Delegate the call to before_call, if the time out is not elapsed it will throw an exception, otherwise we get the results from the call performed after the state is switch to half-open. """ return self.before_call(func, *args, **kwargs) class CircuitHalfOpenState(CircuitBreakerState): """In the "half-open" state, the next call to the circuit breaker is allowed to execute the dangerous operation. Should the call succeed, the circuit breaker resets and returns to the "closed" state. If this trial call fails, however, the circuit breaker returns to the "open" state until another timeout elapses. """ def __init__( self, cb: CircuitBreaker, prev_state: CircuitBreakerState | None, notify: bool = False, ) -> None: """Move the given circuit breaker `cb` to the "half-open" state.""" super().__init__(cb, STATE_HALF_OPEN) if notify: # Reset success counter when entering half-open state self._breaker._state_storage.reset_success_counter() for listener in self._breaker._listeners: listener.state_change(self._breaker, prev_state, self) def on_failure(self, exc: BaseException) -> NoReturn: """Opens the circuit breaker.""" throw_new_error = self._breaker.open() if throw_new_error: error_msg = "Trial call failed, circuit breaker opened" raise CircuitBreakerError(error_msg).with_traceback(sys.exc_info()[2]) raise exc def on_success(self) -> None: """Increment success counter and close the circuit breaker if threshold is reached.""" self._breaker._state_storage.increment_success_counter() if self._breaker._state_storage.success_counter >= self._breaker.success_threshold: self._breaker.close() class CircuitBreakerError(Exception): """When calls to a service fails because the circuit is open, this error is raised to allow the caller to handle this type of exception differently. """ danielfm-pybreaker-7c96106/src/pybreaker/py.typed000066400000000000000000000000001506401210000217610ustar00rootroot00000000000000danielfm-pybreaker-7c96106/tests/000077500000000000000000000000001506401210000166635ustar00rootroot00000000000000danielfm-pybreaker-7c96106/tests/__init__.py000066400000000000000000000000001506401210000207620ustar00rootroot00000000000000danielfm-pybreaker-7c96106/tests/pybreaker_test.py000077500000000000000000001270271506401210000222740ustar00rootroot00000000000000import unittest from contextlib import contextmanager from datetime import datetime from time import sleep from unittest import mock import pytest from pybreaker import * from tornado import gen, testing class CircuitBreakerStorageBasedTestCase: """Mix in to test against different storage backings. Depends on `self.breaker` and `self.breaker_kwargs`. """ def __init__(self): self.breaker_kwargs = None def test_successful_call(self): """CircuitBreaker: it should keep the circuit closed after a successful call. """ def func(): return True assert self.breaker.call(func) assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "closed" def test_one_failed_call(self): """CircuitBreaker: it should keep the circuit closed after a few failures. """ def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) assert self.breaker.fail_counter == 1 assert self.breaker.current_state == "closed" def test_one_successful_call_after_failed_call(self): """CircuitBreaker: it should keep the circuit closed after few mixed outcomes. """ def suc(): return True def err(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(err) assert self.breaker.fail_counter == 1 assert self.breaker.call(suc) assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "closed" def test_several_failed_calls_setting_absent(self): """CircuitBreaker: it should open the circuit after many failures.""" self.breaker = CircuitBreaker(fail_max=3, **self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" def test_throw_new_error_on_trip_false(self): """CircuitBreaker: it should throw the original exception""" self.breaker = CircuitBreaker(fail_max=3, **self.breaker_kwargs, throw_new_error_on_trip=False) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) # Circuit should be open assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" # Circuit should still be open and break with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" def test_throw_new_error_on_trip_true(self): """CircuitBreaker: it should throw a CircuitBreakerError exception""" self.breaker = CircuitBreaker(fail_max=3, **self.breaker_kwargs, throw_new_error_on_trip=True) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" def test_traceback_in_circuitbreaker_error(self): """CircuitBreaker: it should open the circuit after many failures.""" self.breaker = CircuitBreaker(fail_max=3, **self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) # Circuit should open try: self.breaker.call(func) pytest.fail("CircuitBreakerError should throw") except CircuitBreakerError: import traceback assert "NotImplementedError" in traceback.format_exc() assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" def test_failed_call_after_timeout(self): """CircuitBreaker: it should half-open the circuit after timeout.""" self.breaker = CircuitBreaker(fail_max=3, reset_timeout=0.5, **self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) assert self.breaker.current_state == "closed" # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 3 # Wait for timeout sleep(0.6) # Circuit should open again with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 4 assert self.breaker.current_state == "open" def test_successful_after_timeout(self): """CircuitBreaker: it should close the circuit when a call succeeds after timeout. The successful function should only be called once. """ self.breaker = CircuitBreaker(fail_max=3, reset_timeout=1, **self.breaker_kwargs) suc = mock.MagicMock(return_value=True) def err(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(err) with pytest.raises(NotImplementedError): self.breaker.call(err) assert self.breaker.current_state == "closed" # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(err) with pytest.raises(CircuitBreakerError): self.breaker.call(suc) assert self.breaker.fail_counter == 3 # Wait for timeout, at least a second since redis rounds to a second sleep(2) # Circuit should close again assert self.breaker.call(suc) assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "closed" assert suc.call_count == 1 def test_failed_call_when_halfopen(self): """CircuitBreaker: it should open the circuit when a call fails in half-open state. """ def fun(): raise NotImplementedError self.breaker.half_open() assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "half-open" # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(fun) assert self.breaker.fail_counter == 1 assert self.breaker.current_state == "open" def test_successful_call_when_halfopen(self): """CircuitBreaker: it should close the circuit when a call succeeds in half-open state. """ def fun(): return True self.breaker.half_open() assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "half-open" # Circuit should open assert self.breaker.call(fun) assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "closed" def test_close(self): """CircuitBreaker: it should allow the circuit to be closed manually.""" self.breaker = CircuitBreaker(fail_max=3, **self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) with pytest.raises(NotImplementedError): self.breaker.call(func) # Circuit should open with pytest.raises(CircuitBreakerError): self.breaker.call(func) with pytest.raises(CircuitBreakerError): self.breaker.call(func) assert self.breaker.fail_counter == 3 assert self.breaker.current_state == "open" # Circuit should close again self.breaker.close() assert self.breaker.fail_counter == 0 assert self.breaker.current_state == "closed" def test_transition_events(self): """CircuitBreaker: it should call the appropriate functions on every state transition. """ class Listener(CircuitBreakerListener): def __init__(self): self.out = "" def state_change(self, cb, old_state, new_state): assert cb if old_state: self.out += old_state.name if new_state: self.out += "->" + new_state.name self.out += "," listener = Listener() self.breaker = CircuitBreaker(listeners=(listener,), **self.breaker_kwargs) assert self.breaker.current_state == "closed" self.breaker.open() assert self.breaker.current_state == "open" self.breaker.half_open() assert self.breaker.current_state == "half-open" self.breaker.close() assert self.breaker.current_state == "closed" assert listener.out == "closed->open,open->half-open,half-open->closed," def test_call_events(self): """CircuitBreaker: it should call the appropriate functions on every successful/failed call. """ self.out = "" def suc(): return True def err(): raise NotImplementedError class Listener(CircuitBreakerListener): def __init__(self): self.out = "" def before_call(self, cb, func, *args, **kwargs): assert cb self.out += "-" def success(self, cb): assert cb self.out += "success" def failure(self, cb, exc): assert cb assert exc self.out += "failure" listener = Listener() self.breaker = CircuitBreaker(listeners=(listener,), **self.breaker_kwargs) assert self.breaker.call(suc) with pytest.raises(NotImplementedError): self.breaker.call(err) assert listener.out == "-success-failure" def test_generator(self): """CircuitBreaker: it should inspect generator values.""" @self.breaker def suc(value): "Docstring" yield value @self.breaker def err(value): "Docstring" x = yield value raise NotImplementedError(x) s = suc(True) e = err(True) next(e) with pytest.raises(NotImplementedError): e.send(True) assert self.breaker.fail_counter == 1 assert next(s) with pytest.raises((StopIteration, RuntimeError)): next(s) assert self.breaker.fail_counter == 0 def test_contextmanager(self): """CircuitBreaker: it should catch in a with statement""" class Foo: @contextmanager @self.breaker def wrapper(self): try: yield except NotImplementedError: raise ValueError def foo(self): with self.wrapper(): raise NotImplementedError try: Foo().foo() except ValueError as e: assert isinstance(e, ValueError) class CircuitBreakerConfigurationTestCase: """Tests for the CircuitBreaker class.""" def test_default_state(self): """CircuitBreaker: it should get initial state from state_storage.""" for state in (STATE_OPEN, STATE_CLOSED, STATE_HALF_OPEN): storage = CircuitMemoryStorage(state) breaker = CircuitBreaker(state_storage=storage) assert breaker.state.name == state def test_default_params(self): """CircuitBreaker: it should define smart defaults.""" assert self.breaker.fail_counter == 0 assert self.breaker.reset_timeout == 60 assert self.breaker.fail_max == 5 assert self.breaker.current_state == "closed" assert self.breaker.excluded_exceptions == () assert self.breaker.listeners == () assert self.breaker._state_storage.name == "memory" def test_new_with_custom_reset_timeout(self): """CircuitBreaker: it should support a custom reset timeout value.""" self.breaker = CircuitBreaker(reset_timeout=30) assert self.breaker.fail_counter == 0 assert self.breaker.reset_timeout == 30 assert self.breaker.fail_max == 5 assert self.breaker.excluded_exceptions == () assert self.breaker.listeners == () assert self.breaker._state_storage.name == "memory" def test_new_with_custom_fail_max(self): """CircuitBreaker: it should support a custom maximum number of failures. """ self.breaker = CircuitBreaker(fail_max=10) assert self.breaker.fail_counter == 0 assert self.breaker.reset_timeout == 60 assert self.breaker.fail_max == 10 assert self.breaker.excluded_exceptions == () assert self.breaker.listeners == () assert self.breaker._state_storage.name == "memory" def test_new_with_custom_excluded_exceptions(self): """CircuitBreaker: it should support a custom list of excluded exceptions. """ self.breaker = CircuitBreaker(exclude=[Exception]) assert self.breaker.fail_counter == 0 assert self.breaker.reset_timeout == 60 assert self.breaker.fail_max == 5 assert (Exception,) == self.breaker.excluded_exceptions assert self.breaker.listeners == () assert self.breaker._state_storage.name == "memory" def test_fail_max_setter(self): """CircuitBreaker: it should allow the user to set a new value for 'fail_max'. """ assert self.breaker.fail_max == 5 self.breaker.fail_max = 10 assert self.breaker.fail_max == 10 def test_reset_timeout_setter(self): """CircuitBreaker: it should allow the user to set a new value for 'reset_timeout'. """ assert self.breaker.reset_timeout == 60 self.breaker.reset_timeout = 30 assert self.breaker.reset_timeout == 30 def test_call_with_no_args(self): """CircuitBreaker: it should be able to invoke functions with no-args.""" def func(): return True assert self.breaker.call(func) def test_call_with_args(self): """CircuitBreaker: it should be able to invoke functions with args.""" def func(arg1, arg2): return [arg1, arg2] assert [42, "abc"], self.breaker.call(func, 42 == "abc") def test_call_with_kwargs(self): """CircuitBreaker: it should be able to invoke functions with kwargs.""" def func(**kwargs): return kwargs assert {"a": 1, "b": 2}, self.breaker.call(func, a=1, b=2) @testing.gen_test def test_call_async_with_no_args(self): """CircuitBreaker: it should be able to invoke async functions with no-args.""" @gen.coroutine def func(): return True ret = yield self.breaker.call(func) assert ret @testing.gen_test def test_call_async_with_args(self): """CircuitBreaker: it should be able to invoke async functions with args.""" @gen.coroutine def func(arg1, arg2): return [arg1, arg2] ret = yield self.breaker.call(func, 42, "abc") assert [42, "abc"] == ret @testing.gen_test def test_call_async_with_kwargs(self): """CircuitBreaker: it should be able to invoke async functions with kwargs.""" @gen.coroutine def func(**kwargs): return kwargs ret = yield self.breaker.call(func, a=1, b=2) assert {"a": 1, "b": 2} == ret def test_add_listener(self): """CircuitBreaker: it should allow the user to add a listener at a later time. """ assert self.breaker.listeners == () first = CircuitBreakerListener() self.breaker.add_listener(first) assert (first,) == self.breaker.listeners second = CircuitBreakerListener() self.breaker.add_listener(second) assert (first, second) == self.breaker.listeners def test_add_listeners(self): """CircuitBreaker: it should allow the user to add listeners at a later time. """ first, second = CircuitBreakerListener(), CircuitBreakerListener() self.breaker.add_listeners(first, second) assert (first, second) == self.breaker.listeners def test_remove_listener(self): """CircuitBreaker: it should allow the user to remove a listener.""" first = CircuitBreakerListener() self.breaker.add_listener(first) assert (first,) == self.breaker.listeners self.breaker.remove_listener(first) assert self.breaker.listeners == () def test_excluded_exceptions(self): """CircuitBreaker: it should ignore specific exceptions.""" self.breaker = CircuitBreaker(exclude=[LookupError]) def err_1(): raise NotImplementedError def err_2(): raise LookupError def err_3(): raise KeyError with pytest.raises(NotImplementedError): self.breaker.call(err_1) assert self.breaker.fail_counter == 1 # LookupError is not considered a system error with pytest.raises(LookupError): self.breaker.call(err_2) assert self.breaker.fail_counter == 0 with pytest.raises(NotImplementedError): self.breaker.call(err_1) assert self.breaker.fail_counter == 1 # Should consider subclasses as well (KeyError is a subclass of # LookupError) with pytest.raises(KeyError): self.breaker.call(err_3) assert self.breaker.fail_counter == 0 def test_excluded_callable_exceptions(self): """CircuitBreaker: it should ignore specific exceptions that return true from a filtering callable.""" class TestException(Exception): def __init__(self, value): self.value = value filter_function = lambda e: type(e) == TestException and e.value == "good" self.breaker = CircuitBreaker(exclude=[filter_function]) def err_1(): raise TestException("bad") def err_2(): raise TestException("good") def err_3(): raise NotImplementedError with pytest.raises(TestException): self.breaker.call(err_1) assert self.breaker.fail_counter == 1 with pytest.raises(TestException): self.breaker.call(err_2) assert self.breaker.fail_counter == 0 with pytest.raises(NotImplementedError): self.breaker.call(err_3) assert self.breaker.fail_counter == 1 def test_excluded_callable_and_types_exceptions(self): """CircuitBreaker: it should allow a mix of exclusions that includes both filter functions and types.""" class TestException(Exception): def __init__(self, value): self.value = value filter_function = lambda e: type(e) == TestException and e.value == "good" self.breaker = CircuitBreaker(exclude=[filter_function, LookupError]) def err_1(): raise TestException("bad") def err_2(): raise TestException("good") def err_3(): raise NotImplementedError def err_4(): raise LookupError with pytest.raises(TestException): self.breaker.call(err_1) assert self.breaker.fail_counter == 1 with pytest.raises(TestException): self.breaker.call(err_2) assert self.breaker.fail_counter == 0 with pytest.raises(NotImplementedError): self.breaker.call(err_3) assert self.breaker.fail_counter == 1 with pytest.raises(LookupError): self.breaker.call(err_4) assert self.breaker.fail_counter == 0 def test_add_excluded_exception(self): """CircuitBreaker: it should allow the user to exclude an exception at a later time. """ assert self.breaker.excluded_exceptions == () self.breaker.add_excluded_exception(NotImplementedError) assert (NotImplementedError,) == self.breaker.excluded_exceptions self.breaker.add_excluded_exception(Exception) assert (NotImplementedError, Exception) == self.breaker.excluded_exceptions def test_add_excluded_exceptions(self): """CircuitBreaker: it should allow the user to exclude exceptions at a later time. """ self.breaker.add_excluded_exceptions(NotImplementedError, Exception) assert (NotImplementedError, Exception) == self.breaker.excluded_exceptions def test_remove_excluded_exception(self): """CircuitBreaker: it should allow the user to remove an excluded exception. """ self.breaker.add_excluded_exception(NotImplementedError) assert (NotImplementedError,) == self.breaker.excluded_exceptions self.breaker.remove_excluded_exception(NotImplementedError) assert self.breaker.excluded_exceptions == () def test_decorator(self): """CircuitBreaker: it should be a decorator.""" @self.breaker def suc(value): "Docstring" return value @self.breaker def err(value): "Docstring" raise NotImplementedError assert suc.__doc__ == "Docstring" assert err.__doc__ == "Docstring" assert suc.__name__ == "suc" assert err.__name__ == "err" with pytest.raises(NotImplementedError): err(True) assert self.breaker.fail_counter == 1 assert suc(True) assert self.breaker.fail_counter == 0 @testing.gen_test def test_decorator_call_future(self): """CircuitBreaker: it should be a decorator.""" @self.breaker(__pybreaker_call_async=True) @gen.coroutine def suc(value): "Docstring" raise gen.Return(value) @self.breaker(__pybreaker_call_async=True) @gen.coroutine def err(value): "Docstring" raise NotImplementedError assert suc.__doc__ == "Docstring" assert err.__doc__ == "Docstring" assert suc.__name__ == "suc" assert err.__name__ == "err" with pytest.raises(NotImplementedError): yield err(True) assert self.breaker.fail_counter == 1 ret = yield suc(True) assert ret assert self.breaker.fail_counter == 0 @mock.patch("pybreaker.HAS_TORNADO_SUPPORT", False) def test_no_tornado_raises(self): with pytest.raises(ImportError): def func(): return True self.breaker(func, __pybreaker_call_async=True) def test_name(self): """CircuitBreaker: it should allow an optional name to be set and retrieved. """ name = "test_breaker" self.breaker = CircuitBreaker(name=name) assert self.breaker.name == name name = "breaker_test" self.breaker.name = name assert self.breaker.name == name def test_success_threshold_default_behavior(self): """CircuitBreaker: it should maintain backward compatibility with default success_threshold=1.""" self.breaker = CircuitBreaker(fail_max=3, reset_timeout=0.1, **self.breaker_kwargs) def fun(): return True def err(): raise NotImplementedError # Open the circuit for i in range(3): if i < 2: with pytest.raises(NotImplementedError): self.breaker.call(err) else: with pytest.raises(CircuitBreakerError): self.breaker.call(err) # Wait for timeout to enter half-open state sleep(0.2) # Single successful call should close the circuit (default behavior) assert self.breaker.call(fun) assert self.breaker.current_state == "closed" assert self.breaker.success_counter == 0 # Should be reset when closed def test_success_threshold_multiple_successes_required(self): """CircuitBreaker: it should require multiple successful calls before closing when success_threshold > 1.""" self.breaker = CircuitBreaker(fail_max=3, reset_timeout=0.1, success_threshold=3, **self.breaker_kwargs) def fun(): return True def err(): raise NotImplementedError # Open the circuit for i in range(3): if i < 2: with pytest.raises(NotImplementedError): self.breaker.call(err) else: with pytest.raises(CircuitBreakerError): self.breaker.call(err) # Wait for timeout to enter half-open state sleep(0.2) # First successful call should not close the circuit assert self.breaker.call(fun) assert self.breaker.current_state == "half-open" assert self.breaker.success_counter == 1 # Second successful call should not close the circuit assert self.breaker.call(fun) assert self.breaker.current_state == "half-open" assert self.breaker.success_counter == 2 # Third successful call should close the circuit assert self.breaker.call(fun) assert self.breaker.current_state == "closed" assert self.breaker.success_counter == 0 # Should be reset when closed def test_success_threshold_failure_resets_counter(self): """CircuitBreaker: it should reset success counter when a failure occurs in half-open state.""" self.breaker = CircuitBreaker(fail_max=3, reset_timeout=0.1, success_threshold=3, **self.breaker_kwargs) def fun(): return True def err(): raise NotImplementedError # Open the circuit for i in range(3): if i < 2: with pytest.raises(NotImplementedError): self.breaker.call(err) else: with pytest.raises(CircuitBreakerError): self.breaker.call(err) # Wait for timeout to enter half-open state sleep(0.2) # First successful call assert self.breaker.call(fun) assert self.breaker.current_state == "half-open" assert self.breaker.success_counter == 1 # Second successful call assert self.breaker.call(fun) assert self.breaker.current_state == "half-open" assert self.breaker.success_counter == 2 # Failure should reset success counter and open circuit with pytest.raises(CircuitBreakerError): self.breaker.call(err) assert self.breaker.current_state == "open" assert self.breaker.success_counter == 0 # Should be reset when opened def test_success_threshold_property(self): """CircuitBreaker: it should support getting and setting success_threshold.""" self.breaker = CircuitBreaker(success_threshold=5) assert self.breaker.success_threshold == 5 self.breaker.success_threshold = 10 assert self.breaker.success_threshold == 10 def test_success_threshold_redis_storage(self): """CircuitBreaker: it should work correctly with Redis storage.""" # This test requires Redis to be running try: import redis redis_client = redis.Redis() redis_client.ping() except (ImportError, redis.ConnectionError): pytest.skip("Redis not available") storage = CircuitRedisStorage(STATE_CLOSED, redis_client) self.breaker = CircuitBreaker(fail_max=3, reset_timeout=0.1, success_threshold=2, state_storage=storage) def fun(): return True def err(): raise NotImplementedError # Open the circuit for i in range(3): if i < 2: with pytest.raises(NotImplementedError): self.breaker.call(err) else: with pytest.raises(CircuitBreakerError): self.breaker.call(err) # Wait for timeout to enter half-open state sleep(0.2) # First successful call should not close the circuit assert self.breaker.call(fun) assert self.breaker.current_state == "half-open" assert self.breaker.success_counter == 1 # Second successful call should close the circuit assert self.breaker.call(fun) assert self.breaker.current_state == "closed" assert self.breaker.success_counter == 0 class CircuitBreakerTestCase( testing.AsyncTestCase, CircuitBreakerStorageBasedTestCase, CircuitBreakerConfigurationTestCase, ): """Tests for the CircuitBreaker class.""" def setUp(self): super(CircuitBreakerTestCase, self).setUp() self.breaker_kwargs = {} self.breaker = CircuitBreaker() def test_create_new_state__bad_state(self): with pytest.raises(ValueError): self.breaker._create_new_state("foo") @mock.patch("pybreaker.CircuitOpenState") def test_notify_not_called_on_init(self, open_state): storage = CircuitMemoryStorage("open") breaker = CircuitBreaker(state_storage=storage) open_state.assert_called_once_with(breaker, prev_state=None, notify=False) @mock.patch("pybreaker.CircuitOpenState") def test_notify_called_on_state_change(self, open_state): storage = CircuitMemoryStorage("closed") breaker = CircuitBreaker(state_storage=storage) prev_state = breaker.state breaker.state = "open" open_state.assert_called_once_with(breaker, prev_state=prev_state, notify=True) def test_failure_count_not_reset_during_creation(self): for state in (STATE_OPEN, STATE_CLOSED, STATE_HALF_OPEN): storage = CircuitMemoryStorage(state) storage.increment_counter() breaker = CircuitBreaker(state_storage=storage) assert breaker.state.name == state assert breaker.fail_counter == 1 def test_state_opened_at_not_reset_during_creation(self): for state in (STATE_OPEN, STATE_CLOSED, STATE_HALF_OPEN): storage = CircuitMemoryStorage(state) now = datetime.now() storage.opened_at = now breaker = CircuitBreaker(state_storage=storage) assert breaker.state.name == state assert storage.opened_at == now import logging import fakeredis from redis.exceptions import RedisError class CircuitBreakerRedisTestCase(unittest.TestCase, CircuitBreakerStorageBasedTestCase): """Tests for the CircuitBreaker class.""" def setUp(self): self.redis = fakeredis.FakeStrictRedis() self.breaker_kwargs = {"state_storage": CircuitRedisStorage("closed", self.redis)} self.breaker = CircuitBreaker(**self.breaker_kwargs) def tearDown(self): self.redis.flushall() def test_namespace(self): self.redis.flushall() self.breaker_kwargs = {"state_storage": CircuitRedisStorage("closed", self.redis, namespace="my_app")} self.breaker = CircuitBreaker(**self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) keys = self.redis.keys() assert len(keys) == 3 # fail_counter, success_counter, state assert keys[0].decode("utf-8").startswith("my_app") assert keys[1].decode("utf-8").startswith("my_app") assert keys[2].decode("utf-8").startswith("my_app") def test_fallback_state(self): logger = logging.getLogger("pybreaker") logger.setLevel(logging.FATAL) self.breaker_kwargs = { "state_storage": CircuitRedisStorage("closed", self.redis, fallback_circuit_state="open") } self.breaker = CircuitBreaker(**self.breaker_kwargs) def func(k): raise RedisError() with mock.patch.object(self.redis, "get", new=func): state = self.breaker.state assert state.name == "open" def test_missing_state(self): """CircuitBreakerRedis: If state on Redis is missing, it should set the fallback circuit state and reset the fail counter to 0. """ self.breaker_kwargs = { "state_storage": CircuitRedisStorage("closed", self.redis, fallback_circuit_state="open") } self.breaker = CircuitBreaker(**self.breaker_kwargs) def func(): raise NotImplementedError with pytest.raises(NotImplementedError): self.breaker.call(func) assert self.breaker.fail_counter == 1 with mock.patch.object(self.redis, "get", new=lambda k: None): state = self.breaker.state assert state.name == "open" assert self.breaker.fail_counter == 0 def test_cluster_mode(self): self.redis.flushall() storage = CircuitRedisStorage(STATE_OPEN, self.redis, namespace="my_app", cluster_mode=True) breaker_kwargs = {"state_storage": storage} now = datetime.now() storage.opened_at = now now_str = now.strftime("%Y-%m-%d-%H:%M:%S") opened_at = storage.opened_at.strftime("%Y-%m-%d-%H:%M:%S") breaker = CircuitBreaker(**breaker_kwargs) assert breaker.state.name == STATE_OPEN assert opened_at == now_str import threading from types import MethodType class CircuitBreakerThreadsTestCase(unittest.TestCase): """Tests to reproduce common synchronization errors on CircuitBreaker class.""" def setUp(self): self.breaker = CircuitBreaker(fail_max=3000, reset_timeout=1) def _start_threads(self, target, n): """Starts `n` threads that calls `target` and waits for them to finish.""" threads = [threading.Thread(target=target) for i in range(n)] [t.start() for t in threads] [t.join() for t in threads] def _mock_function(self, obj, func): """Replaces a bounded function in `self.breaker` by another.""" setattr(obj, func.__name__, MethodType(func, self.breaker)) def test_fail_thread_safety(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions. """ # Create a specific exception to avoid masking other errors class SpecificException(Exception): pass @self.breaker def err(): raise SpecificException def trigger_error(): for n in range(500): try: err() except SpecificException: pass def _inc_counter(self): c = self._state_storage._fail_counter sleep(0.00005) self._state_storage._fail_counter = c + 1 self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) assert self.breaker.fail_counter == 1500 def test_success_thread_safety(self): """CircuitBreaker: it should compute a successful call atomically to avoid race conditions. """ @self.breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, cb): c = 0 if hasattr(cb, "_success_counter"): c = cb._success_counter sleep(0.00005) cb._success_counter = c + 1 self.breaker.add_listener(SuccessListener()) self._start_threads(trigger_success, 3) assert self.breaker._success_counter == 1500 def test_half_open_thread_safety(self): """CircuitBreaker: it should allow only one trial call when the circuit is half-open. """ self.breaker = CircuitBreaker(fail_max=1, reset_timeout=0.01) self.breaker.open() sleep(0.01) @self.breaker def err(): raise Exception def trigger_failure(): try: err() except: pass class StateListener(CircuitBreakerListener): def __init__(self): self._count = 0 def before_call(self, cb, fun, *args, **kwargs): sleep(0.00005) def state_change(self, cb, old_state, new_state): if new_state.name == "half-open": self._count += 1 state_listener = StateListener() self.breaker.add_listener(state_listener) self._start_threads(trigger_failure, 5) assert state_listener._count == 1 def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. """ @self.breaker def err(): raise Exception def trigger_error(): for i in range(2000): try: err() except: pass class SleepListener(CircuitBreakerListener): def before_call(self, cb, func, *args, **kwargs): sleep(0.00005) self.breaker.add_listener(SleepListener()) self._start_threads(trigger_error, 3) assert self.breaker.fail_max == self.breaker.fail_counter class CircuitBreakerRedisConcurrencyTestCase(unittest.TestCase): """Tests to reproduce common concurrency between different machines connecting to redis. This is simulated locally using threads. """ def setUp(self): self.redis = fakeredis.FakeStrictRedis() self.breaker_kwargs = { "fail_max": 3000, "reset_timeout": 1, "state_storage": CircuitRedisStorage("closed", self.redis), } self.breaker = CircuitBreaker(**self.breaker_kwargs) def tearDown(self): self.redis.flushall() def _start_threads(self, target, n): """Starts `n` threads that calls `target` and waits for them to finish.""" threads = [threading.Thread(target=target) for i in range(n)] [t.start() for t in threads] [t.join() for t in threads] def _mock_function(self, obj, func): """Replaces a bounded function in `self.breaker` by another.""" setattr(obj, func.__name__, MethodType(func, self.breaker)) def test_fail_thread_safety(self): """CircuitBreaker: it should compute a failed call atomically to avoid race conditions. """ # Create a specific exception to avoid masking other errors class SpecificException(Exception): pass @self.breaker def err(): raise SpecificException def trigger_error(): for n in range(500): try: err() except SpecificException: pass def _inc_counter(self): sleep(0.00005) self._state_storage.increment_counter() self._mock_function(self.breaker, _inc_counter) self._start_threads(trigger_error, 3) assert self.breaker.fail_counter == 1500 def test_success_thread_safety(self): """CircuitBreaker: it should compute a successful call atomically to avoid race conditions. """ @self.breaker def suc(): return True def trigger_success(): for n in range(500): suc() class SuccessListener(CircuitBreakerListener): def success(self, cb): c = 0 if hasattr(cb, "_success_counter"): c = cb._success_counter sleep(0.00005) cb._success_counter = c + 1 self.breaker.add_listener(SuccessListener()) self._start_threads(trigger_success, 3) assert self.breaker._success_counter == 1500 def test_half_open_thread_safety(self): """CircuitBreaker: it should allow only one trial call when the circuit is half-open. """ self.breaker = CircuitBreaker(fail_max=1, reset_timeout=0.01) self.breaker.open() sleep(0.01) @self.breaker def err(): raise Exception def trigger_failure(): try: err() except: pass class StateListener(CircuitBreakerListener): def __init__(self): self._count = 0 def before_call(self, cb, fun, *args, **kwargs): sleep(0.00005) def state_change(self, cb, old_state, new_state): if new_state.name == "half-open": self._count += 1 state_listener = StateListener() self.breaker.add_listener(state_listener) self._start_threads(trigger_failure, 5) assert state_listener._count == 1 def test_fail_max_thread_safety(self): """CircuitBreaker: it should not allow more failed calls than 'fail_max' setting. Note that with Redis, where we have separate systems incrementing the counter, we can get concurrent updates such that the counter is greater than the 'fail_max' by the number of systems. To prevent this, we'd need to take out a lock amongst all systems before trying the call. """ @self.breaker def err(): raise Exception def trigger_error(): for i in range(2000): try: err() except: pass class SleepListener(CircuitBreakerListener): def before_call(self, cb, func, *args, **kwargs): sleep(0.00005) self.breaker.add_listener(SleepListener()) num_threads = 3 self._start_threads(trigger_error, num_threads) assert self.breaker.fail_counter < self.breaker.fail_max + num_threads class CircuitBreakerContextManagerTestCase(unittest.TestCase): """Tests for the CircuitBreaker class, when used as a context manager.""" def test_calling(self): """Test that the CircuitBreaker calling() API returns a context manager and works as expected.""" class TestError(Exception): pass breaker = CircuitBreaker(fail_max=2, reset_timeout=0.01) mock_fn = mock.MagicMock() def _do_raise(): with breaker.calling(): raise TestError def _do_succeed(): with breaker.calling(): mock_fn() self.assertRaises(TestError, _do_raise) self.assertRaises(CircuitBreakerError, _do_raise) assert breaker.fail_counter == 2 assert breaker.current_state == "open" # Still fails while circuit breaker is open: self.assertRaises(CircuitBreakerError, _do_succeed) mock_fn.assert_not_called() sleep(0.01) _do_succeed() mock_fn.assert_called_once() assert breaker.fail_counter == 0 assert breaker.current_state == "closed" if __name__ == "__main__": unittest.main() danielfm-pybreaker-7c96106/tests/typechecks.py000077500000000000000000000005101506401210000213760ustar00rootroot00000000000000from pybreaker import STATE_CLOSED, CircuitBreaker, CircuitMemoryStorage # issue #90: incorrect typing for exclude argument # this should not give errors in mypy CircuitBreaker( fail_max=1, reset_timeout=1000, exclude=[lambda e: isinstance(e, RuntimeError)], state_storage=CircuitMemoryStorage(STATE_CLOSED), )