pax_global_header00006660000000000000000000000064152421414650014516gustar00rootroot0000000000000052 comment=e541ab50a68eb36a537cae693b000a47acb6e179 WebwareForPython-DBUtils-8de754c/000077500000000000000000000000001524214146500167145ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/.bumpversion.cfg000066400000000000000000000013361524214146500220270ustar00rootroot00000000000000[bumpversion] current_version = 3.2.0 [bumpversion:file:pyproject.toml] search = version = "{current_version}" replace = version = "{new_version}" [bumpversion:file:dbutils/__init__.py] search = __version__ = "{current_version}" replace = __version__ = "{new_version}" [bumpversion:file:README.md] search = The current version {current_version} replace = The current version {new_version} [bumpversion:file:docs/main.rst] search = :Version: {current_version} replace = :Version: {new_version} [bumpversion:file:docs/main.de.rst] search = :Version: {current_version} replace = :Version: {new_version} [bumpversion:file:tox.ini] search = dbutils-{current_version}-py3-none-any.whl replace = dbutils-{new_version}-py3-none-any.whl WebwareForPython-DBUtils-8de754c/.gitattributes000066400000000000000000000006411524214146500216100ustar00rootroot00000000000000* text=auto eol=lf *.bat text eol=crlf *.config text eol=lf *.css text eol=lf *.html text eol=lf *.js text eol=lf *.prefs text *.py text eol=lf *.rst text eol=lf *.sh text eol=lf *.txt text eol=lf *.po text eol=lf *.pot text eol=lf *.styl text eol=lf *.xml text *.gif binary *.ico binary *.jpg binary *.lnk binary *.mo binary *.png binary *.exe binary *.so binary *.ppt binary *.pdf binary *.gz binary *.zip binary WebwareForPython-DBUtils-8de754c/.github/000077500000000000000000000000001524214146500202545ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/.github/workflows/000077500000000000000000000000001524214146500223115ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/.github/workflows/publish_on_pypi.yml000066400000000000000000000015611524214146500262420ustar00rootroot00000000000000name: Publish DBUtils on PyPI on: push: tags: - 'Release-*' jobs: publish: runs-on: ubuntu-latest # authenticate with an OpenID Connect token issued by GitHub # instead of an API token, using the trusted publisher on PyPI # that has been registered for this workflow and environment permissions: id-token: write environment: name: pypi url: https://pypi.org/p/DBUtils steps: - name: Check out the repository uses: actions/checkout@v7 - name: Set up Python uses: actions/setup-python@v7 with: python-version: "3.14" - name: Install build tool run: python -m pip install build --user - name: Build source tarball and wheel run: python -m build - name: Publish distribution to PyPI uses: pypa/gh-action-pypi-publish@release/v1 WebwareForPython-DBUtils-8de754c/.github/workflows/test_with_tox.yml000066400000000000000000000010111524214146500257310ustar00rootroot00000000000000name: Test DBUtils using tox on: [push, pull_request] jobs: test: runs-on: ubuntu-latest strategy: matrix: python: ['3.9', '3.10', '3.11', '3.12', '3.13', '3.14'] steps: - uses: actions/checkout@v7 - name: Setup Python ${{ matrix.python }} uses: actions/setup-python@v7 with: python-version: ${{ matrix.python }} - run: pip install tox - run: tox -e py - if: matrix.python == 3.14 run: TOXENV=ruff,manifest,docs,spell tox WebwareForPython-DBUtils-8de754c/.gitignore000066400000000000000000000002551524214146500207060ustar00rootroot00000000000000*~ *.bak *.default *.egg-info *.log *.patch *.pid *.pstats *.pyc *.pyo *.swp build dist local .idea .vscode .tox .pytest_cache .ruff_cache test.bat MANIFEST Thumbs.db WebwareForPython-DBUtils-8de754c/LICENSE000066400000000000000000000020761524214146500177260ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2025 Christoph Zwerschke Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. WebwareForPython-DBUtils-8de754c/MANIFEST.in000066400000000000000000000004061524214146500204520ustar00rootroot00000000000000include MANIFEST.in include LICENSE include README.md include .bumpversion.cfg include pyproject.toml include tox.ini recursive-include tests *.py recursive-include docs *.rst make.py *.html *.css *.png prune docs/_build global-exclude *.py[co] __pycache__ WebwareForPython-DBUtils-8de754c/README.md000066400000000000000000000011121524214146500201660ustar00rootroot00000000000000DBUtils ======= DBUtils is a suite of tools providing solid, persistent and pooled connections to a database that can be used in all kinds of multi-threaded environments. The suite supports DB-API 2 compliant database interfaces and the classic PyGreSQL interface. The current version 3.2.0 of DBUtils supports Python versions 3.7 to 3.14. **Please have a look at the [changelog](https://webwareforpython.github.io/DBUtils/changelog.html), because there were some breaking changes in version 2.0.** The DBUtils home page can be found at https://webwareforpython.github.io/DBUtils/ WebwareForPython-DBUtils-8de754c/dbutils/000077500000000000000000000000001524214146500203625ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/dbutils/__init__.py000066400000000000000000000001221524214146500224660ustar00rootroot00000000000000"""The DBUtils main package.""" __all__ = ["__version__"] __version__ = "3.2.0" WebwareForPython-DBUtils-8de754c/dbutils/persistent_db.py000066400000000000000000000271751524214146500236150ustar00rootroot00000000000000"""PersistentDB - persistent DB-API 2 connections. Implements steady, thread-affine persistent connections to a database based on an arbitrary DB-API 2 compliant database interface module. This should result in a speedup for persistent applications such as the application server of "Webware for Python," without loss of robustness. Robustness is provided by using "hardened" SteadyDB connections. Even if the underlying database is restarted and all connections are lost, they will be automatically and transparently reopened. However, since you don't want this to happen in the middle of a database transaction, you must explicitly start transactions with the begin() method so that SteadyDB knows that the underlying connection shall not be replaced and errors passed on until the transaction is completed. Measures are taken to make the database connections thread-affine. This means the same thread always uses the same cached connection, and no other thread will use it. So even if the underlying DB-API module is not thread-safe at the connection level this will be no problem here. For best performance, the application server should keep threads persistent. For this, you have to set MinServerThreads = MaxServerThreads in Webware. For the Python DB-API 2 specification, see: https://www.python.org/dev/peps/pep-0249/ For information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: First you need to set up a generator for your kind of database connections by creating an instance of PersistentDB, passing the following parameters: creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module maxusage: the maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse) Whenever the limit is reached, the connection will be reset. setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...]. failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module isfatal: an optional callable deciding whether a given exception is really fatal for the connection and the failover mechanism shall be applied (see the steady_db module for details) ping: an optional flag controlling when connections are checked (0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) By default, connections are checked with the ping() method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ping=1, or a tuple with the flag and one of the latter (see the steady_db module for details). closeable: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored threadlocal: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases) The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms. For instance, if you are using pgdb as your DB-API 2 database module and want every connection to your local database 'mydb' to be reused 1000 times: import pgdb # import used DB-API 2 module from dbutils.persistent_db import PersistentDB persist = PersistentDB(pgdb, 1000, database='mydb') Once you have set up the generator with these parameters, you can request database connections of that kind: db = persist.connection() You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened SteadyDB version of the underlying DB-API 2 connection. Closing a persistent connection with db.close() will be silently ignored since it would be reopened at the next usage anyway and contrary to the intent of having persistent connections. Instead, the connection will be automatically closed when the thread dies. You can change this behavior by setting the closeable parameter. Note that you need to explicitly start transactions by calling the begin() method. This ensures that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being reused by the same thread. By setting the threadlocal parameter to threading.local, getting connections may become a bit faster, but this may not work in all environments (for instance, mod_wsgi is known to cause problems since it clears the threading.local data between requests). Ideas for improvement: * Add a thread for monitoring, restarting (or closing) bad or expired connections (similar to DBConnectionPool/ResourcePool by Warren Smith). * Optionally log usage, bad connections and exceeding of limits. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Based on an idea presented on the Webware developer mailing list by Geoffrey Talvola in July 2005 Licensed under the MIT license. """ from . import __version__ from .steady_db import connect try: # Prefer the pure Python version of threading.local. # The C implementation turned out to be problematic with mod_wsgi, # since it does not keep the thread-local data between requests. from _threading_local import local except ImportError: # Fall back to the default version of threading.local. from threading import local __all__ = ['PersistentDB', 'PersistentDBError', 'NotSupportedError'] class PersistentDBError(Exception): """General PersistentDB error.""" class NotSupportedError(PersistentDBError): """DB-API module not supported by PersistentDB.""" class PersistentDB: """Generator for persistent DB-API 2 connections. After you have created the connection pool, you can use connection() to get thread-affine, steady DB-API 2 connections. """ version = __version__ def __init__( self, creator, maxusage=None, setsession=None, failures=None, ping=1, closeable=False, threadlocal=None, isfatal=None, *args, **kwargs): """Set up the persistent DB-API 2 connection generator. creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module maxusage: maximum number of reuses of a single connection (number of database operations, 0 or None means unlimited) Whenever the limit is reached, the connection will be reset. setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module isfatal: an optional callable that is passed an exception matching the failures and shall return whether that error is really fatal for the connection, i.e. whether the failover mechanism shall be applied. Use this to distinguish errors that merely report a failed statement (such as a deliberate server side statement timeout) from errors that report a broken connection, when the database module maps both onto the same exception class. The callable must tolerate arbitrary exception instances, including instances without args. ping: determines when the connection should be checked (0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) By default, the connection is checked with its ping() method. Since ping() is not part of the DB-API 2 specification, you can also pass an SQL statement such as "select 1" that shall be executed instead, or a callable that is passed the underlying DB-API 2 connection and shall return whether that connection is still alive. Such a check is made as with ping=1, i.e. whenever a connection is requested; if you want it to be made at other times, pass a tuple with one of the integer values above and the SQL statement or the callable, e.g. ping=(4, "select 1"). closeable: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored threadlocal: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases) args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ try: threadsafety = creator.threadsafety except AttributeError: try: threadsafety = creator.dbapi.threadsafety except AttributeError: try: if not callable(creator.connect): raise AttributeError except AttributeError: threadsafety = 1 else: threadsafety = 0 if not threadsafety: raise NotSupportedError("Database module is not thread-safe.") self._creator = creator self._maxusage = maxusage self._setsession = setsession self._failures = failures self._isfatal = isfatal self._ping = ping self._closeable = closeable self._args, self._kwargs = args, kwargs self.thread = (threadlocal or local)() def steady_connection(self): """Get a steady, non-persistent DB-API 2 connection.""" return connect( self._creator, self._maxusage, self._setsession, self._failures, self._ping, self._closeable, self._isfatal, *self._args, **self._kwargs) def connection(self, shareable=False): # noqa: ARG002 """Get a steady, persistent DB-API 2 connection. The shareable parameter exists only for compatibility with the PooledDB connection method. In reality, persistent connections are of course never shared with other threads. """ try: con = self.thread.connection except AttributeError as error: con = self.steady_connection() if not con.threadsafety(): raise NotSupportedError( "Database module is not thread-safe.") from error self.thread.connection = con con._ping_check() return con def dedicated_connection(self): """Alias for connection(shareable=False).""" return self.connection() WebwareForPython-DBUtils-8de754c/dbutils/persistent_pg.py000066400000000000000000000156261524214146500236340ustar00rootroot00000000000000"""PersistentPg - persistent classic PyGreSQL connections. Implements steady, thread-affine persistent connections to a PostgreSQL database using the classic (not DB-API 2 compliant) PyGreSQL API. This should result in a speedup for persistent applications such as the application server of "Webware for Python," without loss of robustness. Robustness is provided by using "hardened" SteadyPg connections. Even if the underlying database is restarted and all connections are lost, they will be automatically and transparently reopened. However, since you don't want this to happen in the middle of a database transaction, you must explicitly start transactions with the begin() method so that SteadyPg knows that the underlying connection shall not be replaced and errors passed on until the transaction is completed. Measures are taken to make the database connections thread-affine. This means the same thread always uses the same cached connection, and no other thread will use it. So the fact that the classic PyGreSQL pg module is not thread-safe at the connection level is no problem here. For best performance, the application server should keep threads persistent. For this, you have to set MinServerThreads = MaxServerThreads in Webware. For more information on PostgreSQL, see: https://www.postgresql.org/ For more information on PyGreSQL, see: http://www.pygresql.org For more information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: First you need to set up a generator for your kind of database connections by creating an instance of PersistentPg, passing the following parameters: maxusage: the maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: An optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...] closeable: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored threadlocal: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases) Additionally, you have to pass the parameters for the actual PostgreSQL connection which are passed via PyGreSQL, such as the names of the host, database, user, password etc. For instance, if you want every connection to your local database 'mydb' to be reused 1000 times: from dbutils.persistent_pg import PersistentPg persist = PersistentPg(5, dbname='mydb') Once you have set up the generator with these parameters, you can request database connections of that kind: db = persist.connection() You can use these connections just as if they were ordinary classic PyGreSQL API connections. Actually what you get is the hardened SteadyPg version of a classic PyGreSQL connection. Closing a persistent connection with db.close() will be silently ignored since it would be reopened at the next usage anyway and contrary to the intent of having persistent connections. Instead, the connection will be automatically closed when the thread dies. You can change this behavior by setting the closeable parameter. Note that you need to explicitly start transactions by calling the begin() method. This ensures that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being reused in the same thread. To end transactions, use one of the end(), commit() or rollback() methods. By setting the threadlocal parameter to threading.local, getting connections may become a bit faster, but this may not work in all environments (for instance, mod_wsgi is known to cause problems since it clears the threading.local data between requests). Ideas for improvement: * Add a thread for monitoring, restarting (or closing) bad or expired connections (similar to DBConnectionPool/ResourcePool by Warren Smith). * Optionally log usage, bad connections and exceeding of limits. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Based on an idea presented on the Webware developer mailing list by Geoffrey Talvola in July 2005 Licensed under the MIT license. """ from . import __version__ from .steady_pg import SteadyPgConnection try: # Prefer the pure Python version of threading.local. # The C implementation turned out to be problematic with mod_wsgi, # since it does not keep the thread-local data between requests. from _threading_local import local except ImportError: # Fall back to the default version of threading.local. from threading import local __all__ = ['PersistentPg'] class PersistentPg: """Generator for persistent classic PyGreSQL connections. After you have created the connection pool, you can use connection() to get thread-affine, steady PostgreSQL connections. """ version = __version__ def __init__( self, maxusage=None, setsession=None, closeable=False, threadlocal=None, *args, **kwargs): """Set up the persistent PostgreSQL connection generator. maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] closeable: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored threadlocal: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases) args, kwargs: the parameters that shall be used to establish the PostgreSQL connections using class PyGreSQL pg.DB() """ self._maxusage = maxusage self._setsession = setsession self._closeable = closeable self._args, self._kwargs = args, kwargs self.thread = (threadlocal or local)() def steady_connection(self): """Get a steady, non-persistent PyGreSQL connection.""" return SteadyPgConnection( self._maxusage, self._setsession, self._closeable, *self._args, **self._kwargs) def connection(self): """Get a steady, persistent PyGreSQL connection.""" try: con = self.thread.connection except AttributeError: con = self.steady_connection() self.thread.connection = con return con WebwareForPython-DBUtils-8de754c/dbutils/pooled_db.py000066400000000000000000000573071524214146500226770ustar00rootroot00000000000000"""PooledDB - pooling for DB-API 2 connections. Implements a pool of steady, thread-safe cached connections to a database which are transparently reused, using an arbitrary DB-API 2 compliant database interface module. This should result in a speedup for persistent applications such as the application server of "Webware for Python," without loss of robustness. Robustness is provided by using "hardened" SteadyDB connections. Even if the underlying database is restarted and all connections are lost, they will be automatically and transparently reopened. However, since you don't want this to happen in the middle of a database transaction, you must explicitly start transactions with the begin() method so that SteadyDB knows that the underlying connection shall not be replaced and errors passed on until the transaction is completed. Measures are taken to make the pool of connections thread-safe. If the underlying DB-API module is thread-safe at the connection level, the requested connections may be shared with other threads by default, but you can also request dedicated connections in case you need them. For the Python DB-API 2 specification, see: https://www.python.org/dev/peps/pep-0249/ For information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: First you need to set up the database connection pool by creating an instance of PooledDB, passing the following parameters: creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module mincached: the initial number of idle connections in the pool (the default of 0 means no connections are made at startup) maxcached: the maximum number of idle connections in the pool (the default value of 0 or None means unlimited pool size) maxshared: maximum number of shared connections allowed (the default value of 0 or None means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. maxconnections: maximum number of connections generally allowed (the default value of 0 or None means any number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, but by default an error will be reported) maxusage: maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...] reset: how connections should be reset when returned to the pool (False or None to rollback transactions started with begin(), the default value True always issues a rollback for safety's sake) failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module isfatal: an optional callable deciding whether a given exception is really fatal for the connection and the failover mechanism shall be applied (see the steady_db module for details) ping: an optional flag controlling when connections are checked (0 = None = never, 1 = default = whenever fetched from the pool, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) By default, connections are checked with the ping() method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ping=1, or a tuple with the flag and one of the latter (see the steady_db module for details). The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms. For instance, if you are using pgdb as your DB-API 2 database module and want a pool of at least five connections to your local database 'mydb': import pgdb # import used DB-API 2 module from dbutils.pooled_db import PooledDB pool = PooledDB(pgdb, 5, database='mydb') Once you have set up the connection pool you can request database connections from that pool: db = pool.connection() You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened SteadyDB version of the underlying DB-API 2 connection. Please note that the connection may be shared with other threads by default if you set a non-zero maxshared parameter and the DB-API 2 module allows this. If you want to have a dedicated connection, use: db = pool.connection(shareable=False) You can also use this to get a dedicated connection: db = pool.dedicated_connection() If you don't need it anymore, you should immediately return it to the pool with db.close(). You can get another connection in the same way. Warning: In a threaded environment, never do the following: pool.connection().cursor().execute(...) This would release the connection too early for reuse which may be fatal if the connections are not thread-safe. Make sure that the connection object stays alive as long as you are using it, like that: db = pool.connection() cur = db.cursor() cur.execute(...) res = cur.fetchone() cur.close() # or del cur db.close() # or del db You can also use context managers for simpler code: with pool.connection() as db: with db.cursor as cur: cur.execute(...) res = cur.fetchone() Note that you need to explicitly start transactions by calling the begin() method. This ensures that the connection will not be shared with other threads, that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being given back to the connection pool. Ideas for improvement: * Add a thread for monitoring, restarting (or closing) bad or expired connections (similar to DBConnectionPool/ResourcePool by Warren Smith). * Optionally log usage, bad connections and exceeding of limits. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Based on the code of DBPool, contributed to Webware for Python by Dan Green in December 2000 Licensed under the MIT license. """ from contextlib import suppress from functools import total_ordering from threading import Condition from . import __version__ from .steady_db import connect __all__ = [ 'PooledDB', 'PooledDedicatedDBConnection', 'SharedDBConnection', 'PooledSharedDBConnection', 'PooledDBError', 'InvalidConnectionError', 'NotSupportedError', 'TooManyConnectionsError', ] class PooledDBError(Exception): """General PooledDB error.""" class InvalidConnectionError(PooledDBError): """Database connection is invalid.""" class NotSupportedError(PooledDBError): """DB-API module not supported by PooledDB.""" class TooManyConnectionsError(PooledDBError): """Too many database connections were opened.""" # deprecated alias names for error classes InvalidConnection = InvalidConnectionError TooManyConnections = TooManyConnectionsError class PooledDB: """Pool for DB-API 2 connections. After you have created the connection pool, you can use connection() to get pooled, steady DB-API 2 connections. """ version = __version__ def __init__( self, creator, mincached=0, maxcached=0, maxshared=0, maxconnections=0, blocking=False, maxusage=None, setsession=None, reset=True, failures=None, ping=1, isfatal=None, *args, **kwargs): """Set up the DB-API 2 connection pool. creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module mincached: initial number of idle connections in the pool (0 means no connections are made at startup) maxcached: maximum number of idle connections in the pool (0 or None means unlimited pool size) maxshared: maximum number of shared connections (0 or None means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. maxconnections: maximum number of connections generally allowed (0 or None means an arbitrary number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, otherwise an error will be reported) maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] reset: how connections should be reset when returned to the pool (False or None to rollback transactions started with begin(), True to always issue a rollback for safety's sake) failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module isfatal: an optional callable that is passed an exception matching the failures and shall return whether that error is really fatal for the connection, i.e. whether the failover mechanism shall be applied. Use this to distinguish errors that merely report a failed statement (such as a deliberate server side statement timeout) from errors that report a broken connection, when the database module maps both onto the same exception class. The callable must tolerate arbitrary exception instances, including instances without args. ping: determines when the connection should be checked (0 = None = never, 1 = default = whenever fetched from the pool, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values) By default, the connection is checked with its ping() method. Since ping() is not part of the DB-API 2 specification, you can also pass an SQL statement such as "select 1" that shall be executed instead, or a callable that is passed the underlying DB-API 2 connection and shall return whether that connection is still alive. Such a check is made as with ping=1, i.e. whenever a connection is fetched from the pool; if you want it to be made at other times, pass a tuple with one of the integer values above and the SQL statement or the callable, e.g. ping=(4, "select 1"). args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ try: threadsafety = creator.threadsafety except AttributeError: try: threadsafety = creator.dbapi.threadsafety except AttributeError: try: if not callable(creator.connect): raise AttributeError except AttributeError: threadsafety = 1 else: threadsafety = 0 if not threadsafety: raise NotSupportedError("Database module is not thread-safe.") self._creator = creator self._args, self._kwargs = args, kwargs self._blocking = blocking self._maxusage = maxusage self._setsession = setsession self._reset = reset self._failures = failures self._isfatal = isfatal self._ping = ping if mincached is None: mincached = 0 if maxcached is None: maxcached = 0 if maxconnections is None: maxconnections = 0 if maxcached: maxcached = max(maxcached, mincached) self._maxcached = maxcached else: self._maxcached = 0 if threadsafety > 1 and maxshared: self._maxshared = maxshared self._shared_cache = [] # the cache for shared connections else: self._maxshared = 0 if maxconnections: maxconnections = max(maxconnections, maxcached) maxconnections = max(maxconnections, maxshared) self._maxconnections = maxconnections else: self._maxconnections = 0 self._idle_cache = [] # the actual pool of idle connections self._lock = Condition() self._connections = 0 # Establish an initial number of idle database connections: idle = [self.dedicated_connection() for i in range(mincached)] while idle: idle.pop().close() def steady_connection(self): """Get a steady, unpooled DB-API 2 connection.""" return connect( self._creator, self._maxusage, self._setsession, self._failures, self._ping, True, self._isfatal, *self._args, **self._kwargs) def connection(self, shareable=True): """Get a steady, cached DB-API 2 connection from the pool. If shareable is set and the underlying DB-API 2 allows it, then the connection may be shared with other threads. """ if shareable and self._maxshared: with self._lock: while True: while (not self._shared_cache and self._maxconnections and self._connections >= self._maxconnections): self._wait_lock() if len(self._shared_cache) < self._maxshared: # shared cache is not full, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check this connection con = SharedDBConnection(con) self._connections += 1 break # shared cache full or no more connections allowed self._shared_cache.sort() # least shared connection first # only look at it, but leave it in the shared cache # as long as we may still have to wait for it con = self._shared_cache[0] if con.con._transaction: # do not share connections which are in a transaction, # wait until the situation has changed and start over, # since by then the connection may have become idle # and been removed from the shared cache altogether self._wait_lock() continue del self._shared_cache[0] # get it con.con._ping_check() # check the underlying connection con.share() # increase share of this connection break # put the connection (back) into the shared cache self._shared_cache.append(con) self._lock.notify() con = PooledSharedDBConnection(self, con) else: # try to get a dedicated connection with self._lock: while (self._maxconnections and self._connections >= self._maxconnections): self._wait_lock() # connection limit not reached, get a dedicated connection try: # first try to get it from the idle cache con = self._idle_cache.pop(0) except IndexError: # else get a fresh connection con = self.steady_connection() else: con._ping_check() # check connection con = PooledDedicatedDBConnection(self, con) self._connections += 1 return con def dedicated_connection(self): """Alias for connection(shareable=False).""" return self.connection(False) def unshare(self, con): """Decrease the share of a connection in the shared cache.""" with self._lock: con.unshare() shared = con.shared if not shared: # connection is idle # try to remove it from shared cache with suppress(ValueError): # if pool has already been closed self._shared_cache.remove(con) if not shared: # connection has become idle, self.cache(con.con) # so add it to the idle cache def cache(self, con): """Put a dedicated connection back into the idle cache.""" with self._lock: if not self._maxcached or len(self._idle_cache) < self._maxcached: con._reset(force=self._reset) # rollback possible transaction # the idle cache is not full, so put it there self._idle_cache.append(con) # append it to the idle cache else: # if the idle cache is already full, con.close() # then close the connection self._connections -= 1 self._lock.notify() def close(self): """Close all connections in the pool.""" with self._lock: while self._idle_cache: # close all idle connections con = self._idle_cache.pop(0) with suppress(Exception): con.close() if self._maxshared: # close all shared connections while self._shared_cache: con = self._shared_cache.pop(0).con with suppress(Exception): con.close() self._connections -= 1 self._lock.notify_all() def __del__(self): """Delete the pool.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() except: # noqa: E722, S110 pass def _wait_lock(self): """Wait until notified or report an error.""" if not self._blocking: raise TooManyConnectionsError self._lock.wait() # Auxiliary classes for pooled connections class PooledDedicatedDBConnection: """Auxiliary proxy class for pooled dedicated connections.""" def __init__(self, pool, con): """Create a pooled dedicated connection. pool: the corresponding PooledDB instance con: the underlying SteadyDB connection """ # basic initialization to make finalizer work self._con = None # proper initialization of the connection if not con.threadsafety(): raise NotSupportedError("Database module is not thread-safe.") self._pool = pool self._con = con def close(self): """Close the pooled dedicated connection.""" # Instead of actually closing the connection, # return it to the pool for future reuse. if self._con: self._pool.cache(self._con) self._con = None def __getattr__(self, name): """Proxy all members of the class.""" if self._con: return getattr(self._con, name) raise InvalidConnectionError def __del__(self): """Delete the pooled connection.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() except: # noqa: E722, S110 pass def __enter__(self): """Enter a runtime context for the connection.""" return self def __exit__(self, *exc): """Exit a runtime context for the connection.""" self.close() @total_ordering class SharedDBConnection: """Auxiliary class for shared connections.""" def __init__(self, con): """Create a shared connection. con: the underlying SteadyDB connection """ self.con = con self.shared = 1 def __lt__(self, other): """Check whether this connection should come before the other one.""" if self.con._transaction == other.con._transaction: return self.shared < other.shared return not self.con._transaction def __eq__(self, other): """Check whether this connection is the same as the other one.""" # The ordering above only serves to pick the least shared connection, # it does not make different connections interchangeable. Therefore # equality is identity, so that the shared cache can be searched for # one particular connection (see the unshare method of the pool). return self is other def __hash__(self): """Get hash value of this connection.""" # must be based on identity as well, and must stay constant even # though the number of shares of the connection can change return id(self) def share(self): """Increase the share of this connection.""" self.shared += 1 def unshare(self): """Decrease the share of this connection.""" self.shared -= 1 class PooledSharedDBConnection: """Auxiliary proxy class for pooled shared connections.""" def __init__(self, pool, shared_con): """Create a pooled shared connection. pool: the corresponding PooledDB instance con: the underlying SharedDBConnection """ # basic initialization to make finalizer work self._con = None # proper initialization of the connection con = shared_con.con if not con.threadsafety() > 1: raise NotSupportedError("Database connection is not thread-safe.") self._pool = pool self._shared_con = shared_con self._con = con def close(self): """Close the pooled shared connection.""" # Instead of actually closing the connection, # unshare it and/or return it to the pool. if self._con: self._pool.unshare(self._shared_con) self._shared_con = self._con = None def __getattr__(self, name): """Proxy all members of the class.""" if self._con: return getattr(self._con, name) raise InvalidConnectionError def __del__(self): """Delete the pooled connection.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() except: # noqa: E722, S110 pass def __enter__(self): """Enter a runtime context for the connection.""" return self def __exit__(self, *exc): """Exit a runtime context for the connection.""" self.close() WebwareForPython-DBUtils-8de754c/dbutils/pooled_pg.py000066400000000000000000000312271524214146500227110ustar00rootroot00000000000000"""PooledPg - pooling for classic PyGreSQL connections. Implements a pool of steady, thread-safe cached connections to a PostgreSQL database which are transparently reused, using the classic (not DB-API 2 compliant) PyGreSQL API. This should result in a speedup for persistent applications such as the application server of "Webware for Python," without loss of robustness. Robustness is provided by using "hardened" SteadyPg connections. Even if the underlying database is restarted and all connections are lost, they will be automatically and transparently reopened. However, since you don't want this to happen in the middle of a database transaction, you must explicitly start transactions with the begin() method so that SteadyPg knows that the underlying connection shall not be replaced and errors passed on until the transaction is completed. Measures are taken to make the pool of connections thread-safe regardless of the fact that the classic PyGreSQL pg module itself is not thread-safe at the connection level. For more information on PostgreSQL, see: https://www.postgresql.org/ For more information on PyGreSQL, see: http://www.pygresql.org For more information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: First you need to set up the database connection pool by creating an instance of PooledPg, passing the following parameters: mincached: the initial number of connections in the pool (the default of 0 means no connections are made at startup) maxcached: the maximum number of connections in the pool (the default value of 0 or None means unlimited pool size) maxconnections: maximum number of connections generally allowed (the default value of 0 or None means any number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, but by default an error will be reported) maxusage: maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...] Additionally, you have to pass the parameters for the actual PostgreSQL connection which are passed via PyGreSQL, such as the names of the host, database, user, password etc. For instance, if you want a pool of at least five connections to your local database 'mydb': from dbutils.pooled_pg import PooledPg pool = PooledPg(5, dbname='mydb') Once you have set up the connection pool you can request database connections from that pool: db = pool.connection() You can use these connections just as if they were ordinary classic PyGreSQL API connections. Actually what you get is a proxy class for the hardened SteadyPg version of the connection. The connection will not be shared with other threads. If you don't need it anymore, you should immediately return it to the pool with db.close(). You can get another connection in the same way or with db.reopen(). Warning: In a threaded environment, never do the following: res = pool.connection().query(...).getresult() This would release the connection too early for reuse which may be fatal because the connections are not thread-safe. Make sure that the connection object stays alive as long as you are using it, like that: db = pool.connection() res = db.query(...).getresult() db.close() # or del db You can also a context manager for simpler code: with pool.connection() as db: res = db.query(...).getresult() Note that you need to explicitly start transactions by calling the begin() method. This ensures that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being given back to the connection pool. To end transactions, use one of the end(), commit() or rollback() methods. Ideas for improvement: * Add a thread for monitoring, restarting (or closing) bad or expired connections (similar to DBConnectionPool/ResourcePool by Warren Smith). * Optionally log usage, bad connections and exceeding of limits. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Based on the code of DBPool, contributed to Webware for Python by Dan Green in December 2000 Licensed under the MIT license. """ from contextlib import suppress from queue import Empty, Full, Queue from . import __version__ from .steady_pg import SteadyPgConnection __all__ = [ 'PooledPg', 'PooledPgConnection', 'PooledPgError', 'InvalidConnectionError', 'TooManyConnectionsError', 'RESET_ALWAYS_ROLLBACK', 'RESET_COMPLETELY', ] # constants for "reset" parameter RESET_ALWAYS_ROLLBACK = 1 RESET_COMPLETELY = 2 class PooledPgError(Exception): """General PooledPg error.""" class InvalidConnectionError(PooledPgError): """Database connection is invalid.""" class TooManyConnectionsError(PooledPgError): """Too many database connections were opened.""" # deprecated alias names for error classes InvalidConnection = InvalidConnectionError TooManyConnections = TooManyConnectionsError class PooledPg: """Pool for classic PyGreSQL connections. After you have created the connection pool, you can use connection() to get pooled, steady PostgreSQL connections. """ version = __version__ def __init__( self, mincached=0, maxcached=0, maxconnections=0, blocking=False, maxusage=None, setsession=None, reset=None, *args, **kwargs): """Set up the PostgreSQL connection pool. mincached: initial number of connections in the pool (0 means no connections are made at startup) maxcached: maximum number of connections in the pool (0 or None means unlimited pool size) maxconnections: maximum number of connections generally allowed (0 or None means an arbitrary number of connections) blocking: determines behavior when exceeding the maximum (if this is set to true, block and wait until the number of connections decreases, otherwise an error will be reported) maxusage: maximum number of reuses of a single connection (0 or None means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] reset: how connections should be reset when returned to the pool (0 or None to rollback transactions started with begin(), 1 to always issue a rollback, 2 for a complete reset) args, kwargs: the parameters that shall be used to establish the PostgreSQL connections using class PyGreSQL pg.DB() """ self._args, self._kwargs = args, kwargs self._maxusage = maxusage self._setsession = setsession self._reset = reset or 0 if mincached is None: mincached = 0 if maxcached is None: maxcached = 0 if maxconnections is None: maxconnections = 0 if maxcached and maxcached < mincached: maxcached = mincached if maxconnections: maxconnections = max(maxconnections, maxcached) # Create semaphore for number of allowed connections generally: from threading import Semaphore # noqa: PLC0415 self._connections = Semaphore(maxconnections) self._blocking = blocking else: self._connections = None self._cache = Queue(maxcached) # the actual connection pool # Establish an initial number of database connections: idle = [self.connection() for i in range(mincached)] while idle: idle.pop().close() def steady_connection(self): """Get a steady, unpooled PostgreSQL connection.""" return SteadyPgConnection(self._maxusage, self._setsession, True, *self._args, **self._kwargs) def pooled_connection(self): """Get a steady PostgreSQL connection from the pool cache. This is the unwrapped connection that is kept in the pool cache. It reserves one of the generally allowed connections, so it must be balanced with a corresponding call of the cache() method. """ if self._connections and not self._connections.acquire(self._blocking): raise TooManyConnectionsError try: try: return self._cache.get_nowait() except Empty: return self.steady_connection() except Exception: # the connection could not be established, so give back the # reservation that has been made above (note that this must # not happen when the reservation itself has already failed) if self._connections: self._connections.release() raise def connection(self): """Get a steady, cached PostgreSQL connection from the pool.""" return PooledPgConnection(self, self.pooled_connection()) def cache(self, con): """Put a connection back into the pool cache.""" try: if self._reset == RESET_COMPLETELY: con.reset() # reset the connection completely elif self._reset == RESET_ALWAYS_ROLLBACK or con._transaction: with suppress(Exception): con.rollback() # rollback a possible transaction self._cache.put_nowait(con) # and then put it back into the cache except Full: con.close() if self._connections: self._connections.release() def close(self): """Close all connections in the pool.""" # Note that the connections in the cache have already given back # their share of the generally allowed connections when they were # returned to the pool, so the semaphore must not be released here. # Connections that are still in use are not closed here, and they # will release their share as usual by calling the cache() method. while 1: try: con = self._cache.get_nowait() with suppress(Exception): con.close() except Empty: break def __del__(self): """Delete the pool.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() except: # noqa: E722, S110 pass # Auxiliary class for pooled connections class PooledPgConnection: """Proxy class for pooled PostgreSQL connections.""" def __init__(self, pool, con): """Create a pooled DB-API 2 connection. pool: the corresponding PooledPg instance con: the underlying SteadyPg connection """ self._pool = pool self._con = con def close(self): """Close the pooled connection.""" # Instead of actually closing the connection, # return it to the pool so that it can be reused. if self._con: self._pool.cache(self._con) self._con = None def reopen(self): """Reopen the pooled connection.""" # If the connection is already back in the pool, # get another connection from the pool, # otherwise reopen the underlying connection. if self._con: self._con.reopen() else: # take the steady connection itself out of the pool, since # wrapping another pooled connection would nest the proxies # and put a proxy instead of a connection into the pool cache self._con = self._pool.pooled_connection() def __getattr__(self, name): """Proxy all members of the class.""" if self._con: return getattr(self._con, name) raise InvalidConnectionError def __del__(self): """Delete the pooled connection.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() except: # noqa: E722, S110 pass def __enter__(self): """Enter a runtime context for the connection.""" return self def __exit__(self, *exc): """Exit a runtime context for the connection.""" self.close() WebwareForPython-DBUtils-8de754c/dbutils/simple_pooled_db.py000066400000000000000000000205161524214146500242400ustar00rootroot00000000000000"""SimplePooledDB - a very simple DB-API 2 database connection pool. Implements a pool of threadsafe cached DB-API 2 connections to a database which are transparently reused. This should result in a speedup for persistent applications such as the "Webware for Python" AppServer. For more information on the DB-API 2, see: https://www.python.org/dev/peps/pep-0249/ For more information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Measures are taken to make the pool of connections threadsafe regardless of whether the DB-API 2 module used is threadsafe on the connection level (threadsafety > 1) or not. It must only be threadsafe on the module level (threadsafety = 1). If the DB-API 2 module is threadsafe, the connections will be shared between threads (keep this in mind if you use transactions). Usage: The idea behind SimplePooledDB is that it's completely transparent. After you have established your connection pool, stating the DB-API 2 module to be used, the number of connections to be cached in the pool and the connection parameters, e.g. import pgdb # import used DB-API 2 module from dbutils.simple_pooled_db import PooledDB dbpool = PooledDB(pgdb, 5, host=..., database=..., user=..., ...) you can demand database connections from that pool, db = dbpool.connection() and use them just as if they were ordinary DB-API 2 connections. It's really just a proxy class. db.close() will return the connection to the pool, it will not actually close it. This is so your existing code works nicely. Ideas for improvement: * Do not create the maximum number of connections on startup already, but only a certain number and the rest on demand. * Detect and transparently reset "bad" connections. * Connections should have some sort of maximum usage limit after which they should be automatically closed and reopened. * Prefer or enforce thread-affinity for the connections, allowing for both shareable and non-shareable connections. Please note that these and other ideas have been already implemented in in PooledDB, a more sophisticated version of SimplePooledDB. You might also consider using PersistentDB instead for thread-affine persistent database connections. SimplePooledDB may still serve as a very simple reference and example implementation for developers. Copyright, credits and license: * Contributed as MiscUtils/DBPool for Webware for Python by Dan Green, December 2000 * Thread safety bug found by Tom Schwaller * Fixes by Geoff Talvola (thread safety in _threadsafe_getConnection()) * Clean up by Chuck Esterbrook * Fix unthreadsafe functions which were leaking, Jay Love * Eli Green's webware-discuss comments were lifted for additional docs * Clean-up and detailed commenting, rename and move to DBUtils by Christoph Zwerschke in September 2005 Licensed under the MIT license. """ from . import __version__ __all__ = [ 'PooledDB', 'PooledDBConnection', 'PooledDBError', 'NotSupportedError', ] class PooledDBError(Exception): """General PooledDB error.""" class NotSupportedError(PooledDBError): """DB-API module not supported by PooledDB.""" class PooledDBConnection: """A proxy class for pooled database connections. You don't normally deal with this class directly, but use PooledDB to get new connections. """ def __init__(self, pool, con): """Initialize pooled connection.""" self._con = con self._pool = pool def close(self): """Close the pooled connection.""" # Instead of actually closing the connection, # return it to the pool so that it can be reused. if self._con is not None: self._pool.returnConnection(self._con) self._con = None def __getattr__(self, name): """Get the attribute with the given name.""" # All other attributes are the same. return getattr(self._con, name) def __del__(self): """Delete the pooled connection.""" self.close() class PooledDB: """A very simple database connection pool. After you have created the connection pool, you can get connections using getConnection(). """ version = __version__ def __init__(self, dbapi, maxconnections, *args, **kwargs): """Set up the database connection pool. dbapi: the DB-API 2 compliant module you want to use maxconnections: the number of connections cached in the pool args, kwargs: the parameters that shall be used to establish the database connections using connect() """ try: threadsafety = dbapi.threadsafety except Exception: threadsafety = None if threadsafety == 0: raise NotSupportedError( "Database module does not support any level of threading.") if threadsafety == 1: # If there is no connection level safety, build # the pool using the synchronized queue class # that implements all the required locking semantics. from queue import Queue # noqa: PLC0415 self._queue = Queue(maxconnections) # create the queue self.connection = self._unthreadsafe_get_connection self.addConnection = self._unthreadsafe_add_connection self.returnConnection = self._unthreadsafe_return_connection elif threadsafety in (2, 3): # If there is connection level safety, implement the # pool with an ordinary list used as a circular buffer. # We only need a minimum of locking in this case. from threading import Lock # noqa: PLC0415 self._lock = Lock() # create a lock object to be used later self._nextConnection = 0 # index of the next connection to be used self._connections = [] # the list of connections self.connection = self._threadsafe_get_connection self.addConnection = self._threadsafe_add_connection self.returnConnection = self._threadsafe_return_connection else: raise NotSupportedError( "Database module threading support cannot be determined.") # Establish all database connections (it would be better to # only establish a part of them now, and the rest on demand). for _i in range(maxconnections): self.addConnection(dbapi.connect(*args, **kwargs)) # The following functions are used with DB-API 2 modules # that do not have connection level threadsafety, like PyGreSQL. # However, the module must be threadsafe at the module level. # Note: threadsafe/unthreadsafe refers to the DB-API 2 module, # not to this class which should be threadsafe in any case. def _unthreadsafe_get_connection(self): """Get a connection from the pool.""" return PooledDBConnection(self, self._queue.get()) def _unthreadsafe_add_connection(self, con): """Add a connection to the pool.""" self._queue.put(con) def _unthreadsafe_return_connection(self, con): """Return a connection to the pool. In this case, the connections need to be put back into the queue after they have been used. This is done automatically when the connection is closed and should never be called explicitly outside of this module. """ self._unthreadsafe_add_connection(con) # The following functions are used with DB-API 2 modules # that are threadsafe at the connection level, like psycopg. # Note: In this case, connections are shared between threads. # This may lead to problems if you use transactions. def _threadsafe_get_connection(self): """Get a connection from the pool.""" with self._lock: next_con = self._nextConnection con = PooledDBConnection(self, self._connections[next_con]) next_con += 1 if next_con >= len(self._connections): next_con = 0 self._nextConnection = next_con return con def _threadsafe_add_connection(self, con): """Add a connection to the pool.""" self._connections.append(con) def _threadsafe_return_connection(self, con): """Return a connection to the pool. In this case, the connections always stay in the pool, so there is no need to do anything here. """ # we don't need to do anything here WebwareForPython-DBUtils-8de754c/dbutils/simple_pooled_pg.py000066400000000000000000000113021524214146500242520ustar00rootroot00000000000000"""SimplePooledPg - a very simple classic PyGreSQL connection pool. Implements a pool of threadsafe cached connections to a PostgreSQL database which are transparently reused, using the classic (not DB-API 2 compliant) PyGreSQL pg API. This should result in a speedup for persistent applications such as the "Webware for Python" AppServer. For more information on PostgreSQL, see: https://www.postgresql.org/ For more information on PyGreSQL, see: http://www.pygresql.org For more information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Measures are taken to make the pool of connections threadsafe regardless of the fact that the PyGreSQL pg module itself is not threadsafe at the connection level. Connections will never be shared between threads, so you can safely use transactions. Usage: The idea behind SimplePooledPg is that it's completely transparent. After you have established your connection pool, stating the number of connections to be cached in the pool and the connection parameters, e.g. from dbutils.simple_pooled_pg import PooledPg dbpool = PooledPg(5, host=..., database=..., user=..., ...) you can demand database connections from that pool, db = dbpool.connection() and use them just as if they were ordinary PyGreSQL pg API connections. It's really just a proxy class. db.close() will return the connection to the pool, it will not actually close it. This is so your existing code works nicely. Ideas for improvement: * Do not create the maximum number of connections on startup already, but only a certain number and the rest on demand. * Detect and transparently reset "bad" connections. The PyGreSQL pg API provides a status attribute and a reset() method for that. * Connections should have some sort of "maximum usage limit" after which they should be automatically closed and reopened. * Prefer or enforce thread affinity for the connections. Please note that these and other ideas have been already implemented in in PooledPg, a more sophisticated version of SimplePooledPg. You might also consider using PersistentPg instead for thread-affine persistent PyGreSQL connections. SimplePooledPg may still serve as a very simple reference and example implementation for developers. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Based on the code of DBPool, contributed to Webware for Python by Dan Green in December 2000 Licensed under the MIT license. """ from pg import DB as PgConnection # noqa: N811 from . import __version__ __all__ = ['PooledPg', 'PooledPgConnection'] class PooledPgConnection: """A proxy class for pooled PostgreSQL connections. You don't normally deal with this class directly, but use PooledPg to get new connections. """ def __init__(self, pool, con): """Initialize pooled connection.""" self._con = con self._pool = pool def close(self): """Close the pooled connection.""" # Instead of actually closing the connection, # return it to the pool so that it can be reused. if self._con is not None: self._pool.cache(self._con) self._con = None def __getattr__(self, name): """Get the attribute with the given name.""" # All other attributes are the same. return getattr(self._con, name) def __del__(self): """Delete the pooled connection.""" self.close() class PooledPg: """A very simple PostgreSQL connection pool. After you have created the connection pool, you can get connections using getConnection(). """ version = __version__ def __init__(self, maxconnections, *args, **kwargs): """Set up the PostgreSQL connection pool. maxconnections: the number of connections cached in the pool args, kwargs: the parameters that shall be used to establish the PostgreSQL connections using pg.connect() """ # Since there is no connection level safety, we # build the pool using the synchronized queue class # that implements all the required locking semantics. from queue import Queue # noqa: PLC0415 self._queue = Queue(maxconnections) # Establish all database connections (it would be better to # only establish a part of them now, and the rest on demand). for _i in range(maxconnections): self.cache(PgConnection(*args, **kwargs)) def cache(self, con): """Add or return a connection to the pool.""" self._queue.put(con) def connection(self): """Get a connection from the pool.""" return PooledPgConnection(self, self._queue.get()) WebwareForPython-DBUtils-8de754c/dbutils/steady_db.py000066400000000000000000001011531524214146500226730ustar00rootroot00000000000000"""SteadyDB - hardened DB-API 2 connections. Implements steady connections to a database based on an arbitrary DB-API 2 compliant database interface module. The connections are transparently reopened when they are closed or the database connection has been lost or when they are used more often than an optional usage limit. Database cursors are transparently reopened as well when the execution of a database operation cannot be performed due to a lost connection. Only if the connection is lost after the execution, when rows are already fetched from the database, this will give an error and the cursor will not be reopened automatically, because there is no reliable way to recover the state of the cursor in such a situation. Connections which have been marked as being in a transaction with a begin() call will not be silently replaced either. A typical situation where database connections are lost is when the database server or an intervening firewall is shutdown and restarted for maintenance reasons. In such a case, all database connections would become unusable, even though the database service may be already available again. The "hardened" connections provided by this module will make the database connections immediately available again. This approach results in a steady database connection that can be used by PooledDB or PersistentDB to create pooled or persistent connections to a database in a threaded environment such as the application server of "Webware for Python." Note, however, that the connections themselves may not be thread-safe (depending on the used DB-API module). For the Python DB-API 2 specification, see: https://www.python.org/dev/peps/pep-0249/ For information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: You can use the connection constructor connect() in the same way as you would use the connection constructor of a DB-API 2 module if you specify the DB-API 2 module to be used as the first parameter, or alternatively you can specify an arbitrary constructor function returning new DB-API 2 compliant connection objects as the first parameter. Passing just a function allows implementing failover mechanisms and load balancing strategies. You may also specify a usage limit as the second parameter (set it to None if you prefer unlimited usage), an optional list of commands that may serve to prepare the session as a third parameter, the exception classes for which the failover mechanism shall be applied, and you can specify whether is is allowed to close the connection (by default this is true). When the connection to the database is lost or has been used too often, it will be transparently reset in most situations, without further notice. import pgdb # import used DB-API 2 module from dbutils.steady_db import connect db = connect(pgdb, 10000, ["set datestyle to german"], host=..., database=..., user=..., ...) ... cursor = db.cursor() ... cursor.execute('select ...') result = cursor.fetchall() ... cursor.close() ... db.close() Ideas for improvement: * Alternatively to the maximum number of uses, implement a maximum time to live for connections. * Optionally log usage and loss of connection. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 * Allowing creator functions as first parameter as in SQLAlchemy suggested by Ezio Vernacotola in December 2006 Licensed under the MIT license. """ import sys from contextlib import contextmanager, suppress from . import __version__ __all__ = [ 'connect', 'SteadyDBConnection', 'SteadyDBCursor', 'SteadyDBError', 'InvalidCursorError', ] class SteadyDBError(Exception): """General SteadyDB error.""" class InvalidCursorError(SteadyDBError): """Database cursor is invalid.""" # deprecated alias names for error classes InvalidCursor = InvalidCursorError _PING_ERROR = ( "'ping' must be an integer, an SQL statement, a callable," " or a tuple with an integer and one of the latter.") def _sql_pinger(sql): """Create a check function executing the given SQL statement.""" def pinger(con): """Check the connection by executing an SQL statement.""" cursor = con.cursor() try: cursor.execute(sql) finally: with suppress(Exception): cursor.close() return pinger def connect( creator, maxusage=None, setsession=None, failures=None, ping=1, closeable=True, isfatal=None, *args, **kwargs): """Create a "tough" connection. A hardened version of the connection function of a DB-API 2 module. creator: either an arbitrary function returning new DB-API 2 compliant connection objects or a DB-API 2 compliant database module maxusage: maximum usage limit for the underlying DB-API 2 connection (number of database operations, 0 or None means unlimited usage) callproc(), execute() and executemany() count as one operation. When the limit is reached, the connection is automatically reset. setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", "set time zone mez"] failures: an optional exception class or a tuple of exception classes for which the failover mechanism shall be applied, if the default (OperationalError, InternalError, Interface) is not adequate for the used database module isfatal: an optional callable that is passed an exception matching the failures and shall return whether that error is really fatal for the connection, i.e. whether the failover mechanism shall be applied. Since the failures can only be specified as exception classes, and Python cannot exclude subclasses from an except clause, this is the only way to distinguish errors that merely report a failed statement (such as a deliberate server side statement timeout) from errors that report a broken connection, when the database module maps both onto the same exception class. The callable must tolerate arbitrary exception instances of the classes given as failures, including instances without args. ping: determines when and how the connection shall be checked (0 = None = never, 1 = default = when _ping_check() is called, 2 = whenever a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values). By default, the connection is checked with its ping() method. Since ping() is not part of the DB-API 2 specification and not available in every database module, you can also pass an SQL statement such as "select 1" that shall be executed instead, or a callable that is passed the underlying DB-API 2 connection and shall return whether that connection is still alive, where None counts as alive and raising an exception counts as not alive. Such a check is made as with ping=1, i.e. when _ping_check() is called; if you want it to be made at other times, pass a tuple with one of the integer values above and the SQL statement or the callable, e.g. ping=(4, "select 1"). closeable: if this is set to false, then closing the connection will be silently ignored, but by default the connection can be closed args, kwargs: the parameters that shall be passed to the creator function or the connection constructor of the DB-API 2 module """ return SteadyDBConnection( creator, maxusage, setsession, failures, ping, closeable, isfatal, *args, **kwargs) class SteadyDBConnection: """A hardened version of DB-API 2 connections.""" version = __version__ def __init__( self, creator, maxusage=None, setsession=None, failures=None, ping=1, closeable=True, isfatal=None, *args, **kwargs): """Create a "tough" DB-API 2 connection.""" # basic initialization to make finalizer work self._con = None self._closed = True # proper initialization of the connection try: self._creator = creator.connect try: if creator.dbapi.connect: self._dbapi = creator.dbapi except AttributeError: self._dbapi = creator except AttributeError: # try finding the DB-API 2 module via the connection creator self._creator = creator try: self._dbapi = creator.dbapi except AttributeError: try: self._dbapi = sys.modules[creator.__module__] if self._dbapi.connect != creator: raise AttributeError except (AttributeError, KeyError): self._dbapi = None try: self._threadsafety = creator.threadsafety except AttributeError: try: self._threadsafety = self._dbapi.threadsafety except AttributeError: self._threadsafety = None if not callable(self._creator): raise TypeError(f"{creator!r} is not a connection provider.") if maxusage is None: maxusage = 0 if not isinstance(maxusage, int): raise TypeError("'maxusage' must be an integer value.") self._maxusage = maxusage self._setsession_sql = setsession if failures is not None and not isinstance( failures, tuple) and not issubclass(failures, Exception): raise TypeError("'failures' must be a tuple of exceptions.") self._failures = failures if isfatal is not None and not callable(isfatal): raise TypeError("'isfatal' must be a callable.") self._isfatal = isfatal self._ping, self._pinger = self._parse_ping(ping) self._closeable = closeable self._args, self._kwargs = args, kwargs self._store(self._create()) @staticmethod def _parse_ping(ping): """Split the ping parameter into a flag and a check function. The ping parameter determines when the connection shall be checked and can also determine how it shall be checked, by passing an SQL statement or a callable instead of or together with the flag. """ if isinstance(ping, tuple): try: ping, pinger = ping except ValueError: raise TypeError(_PING_ERROR) from None elif isinstance(ping, str) or callable(ping): ping, pinger = 1, ping else: pinger = None if isinstance(pinger, str): pinger = _sql_pinger(pinger) elif pinger is not None and not callable(pinger): raise TypeError(_PING_ERROR) return ping if isinstance(ping, int) else 0, pinger def __enter__(self): """Enter the runtime context for the connection object.""" return self def __exit__(self, *exc): """Exit the runtime context for the connection object. This does not close the connection, but it ends a transaction. """ if exc[0] is None and exc[1] is None and exc[2] is None: self.commit() else: self.rollback() def _create(self): """Create a new connection using the creator function.""" con = self._creator(*self._args, **self._kwargs) try: try: if self._dbapi.connect != self._creator: raise AttributeError except AttributeError: # try finding the DB-API 2 module via the connection itself try: mod = con.__module__ except AttributeError: mod = None while mod: try: self._dbapi = sys.modules[mod] if not callable(self._dbapi.connect): raise AttributeError except (AttributeError, KeyError): pass else: break i = mod.rfind('.') mod = None if i < 0 else mod[:i] else: try: mod = con.OperationalError.__module__ except AttributeError: mod = None while mod: try: self._dbapi = sys.modules[mod] if not callable(self._dbapi.connect): raise AttributeError except (AttributeError, KeyError): pass else: break i = mod.rfind('.') mod = None if i < 0 else mod[:i] else: self._dbapi = None if self._threadsafety is None: try: self._threadsafety = self._dbapi.threadsafety except AttributeError: with suppress(AttributeError): self._threadsafety = con.threadsafety if self._failures is None: try: self._failures = ( self._dbapi.OperationalError, self._dbapi.InterfaceError, self._dbapi.InternalError) except AttributeError: try: self._failures = ( self._creator.OperationalError, self._creator.InterfaceError, self._creator.InternalError) except AttributeError: try: self._failures = ( con.OperationalError, con.InterfaceError, con.InternalError) except AttributeError as error: raise AttributeError( "Could not determine failure exceptions" " (please set failures or creator.dbapi).", ) from error if isinstance(self._failures, tuple): self._failure = self._failures[0] else: self._failure = self._failures self._setsession(con) except Exception as error: # the database module could not be determined # or the session could not be prepared with suppress(Exception): con.close() # close the connection first raise error # re-raise the original error again return con def _setsession(self, con=None): """Execute the SQL commands for session preparation.""" if con is None: con = self._con if self._setsession_sql: cursor = con.cursor() for sql in self._setsession_sql: cursor.execute(sql) cursor.close() def _store(self, con): """Store a database connection for subsequent use.""" self._con = con self._transaction = False self._closed = False self._usage = 0 def _close(self): """Close the tough connection. You can always close a tough connection with this method, and it will not complain if you close it more than once. """ if not self._closed: with suppress(Exception): self._con.close() self._transaction = False self._closed = True def _reset(self, force=False): """Reset a tough connection. Rollback if forced or the connection was in a transaction. """ if not self._closed and (force or self._transaction): with suppress(Exception): self.rollback() def _ping_check(self, ping=1, reconnect=True): """Check whether the connection is still alive. The connection is checked with the check function that has been derived from the ping parameter, or with the ping() method of the underlying connection if no such function has been specified. If the underlying connection is not active and the ping parameter is set accordingly, the connection will be recreated unless the connection is currently inside a transaction. """ if ping & self._ping: if self._pinger is None: try: # if possible, ping the connection try: # pass a reconnect=False flag if this is supported alive = self._con.ping(False) except TypeError: # the reconnect flag is not supported alive = self._con.ping() except (AttributeError, IndexError, TypeError, ValueError): self._ping = 0 # ping() is not available return None # the connection cannot be checked except Exception: alive = False else: try: # check the connection with the given check function alive = self._pinger(self._con) except Exception: alive = False if alive is None: alive = True if alive: reconnect = False if reconnect and not self._transaction: try: # try to reopen the connection con = self._create() except Exception: # noqa: S110 pass else: self._close() self._store(con) alive = True return alive return None def dbapi(self): """Return the underlying DB-API 2 module of the connection.""" if self._dbapi is None: raise AttributeError( "Could not determine DB-API 2 module" " (please set creator.dbapi).") return self._dbapi @property def dbapi_connection(self): """Return the underlying DB-API 2 connection. Note that operations executed directly on this connection are not covered by the failover mechanism and are not counted towards the maximum usage limit of the steady connection. """ return self._con def _fatal_check(self, error): """Check whether the error is fatal for the connection.""" return self._isfatal is None or self._isfatal(error) def threadsafety(self): """Return the thread safety level of the connection.""" if self._threadsafety is None: if self._dbapi is None: raise AttributeError( "Could not determine threadsafety" " (please set creator.dbapi or creator.threadsafety).") return 0 return self._threadsafety def close(self): """Close the tough connection. You are allowed to close a tough connection by default, and it will not complain if you close it more than once. You can disallow closing connections by setting the closeable parameter to something false. In this case, closing tough connections will be silently ignored. """ if self._closeable: self._close() elif self._transaction: self._reset() def begin(self, *args, **kwargs): """Indicate the beginning of a transaction. During a transaction, connections won't be transparently replaced, and all errors will be raised to the application. If the underlying driver supports this method, it will be called with the given parameters (e.g. for distributed transactions). """ self._transaction = True try: begin = self._con.begin except AttributeError: pass else: begin(*args, **kwargs) def commit(self): """Commit any pending transaction.""" self._transaction = False try: self._con.commit() except self._failures as error: # cannot commit try: # try to reopen the connection con = self._create() except Exception: # noqa: S110 pass else: self._close() self._store(con) raise error # re-raise the original error def rollback(self): """Rollback pending transaction.""" self._transaction = False try: self._con.rollback() except self._failures as error: # cannot rollback try: # try to reopen the connection con = self._create() except Exception: # noqa: S110 pass else: self._close() self._store(con) raise error # re-raise the original error def cancel(self): """Cancel a long-running transaction. If the underlying driver supports this method, it will be called. """ self._transaction = False try: cancel = self._con.cancel except AttributeError: pass else: cancel() def ping(self, *args, **kwargs): """Ping connection.""" return self._con.ping(*args, **kwargs) def _cursor(self, *args, **kwargs): """Create a "tough" cursor. This is a hardened version of the method cursor(). """ # The args and kwargs are not part of the standard, # but some database modules seem to use these. transaction = self._transaction if not transaction: self._ping_check(2) overused = False try: # check whether the connection has been used too often if (self._maxusage and self._usage >= self._maxusage and not transaction): overused = True raise self._failure cursor = self._con.cursor(*args, **kwargs) # try to get a cursor except self._failures as error: # error in getting cursor # the connection must always be reset when it is overused, # otherwise the application can veto the failover mechanism if not overused and not self._fatal_check(error): raise try: # try to reopen the connection con = self._create() except Exception: # noqa: S110 pass else: try: # and try one more time to get a cursor cursor = con.cursor(*args, **kwargs) except Exception: # noqa: S110 pass else: self._close() self._store(con) if transaction: raise error # re-raise the original error again return cursor with suppress(Exception): con.close() if transaction: self._transaction = False raise error # re-raise the original error again return cursor def cursor(self, *args, **kwargs): """Return a new Cursor Object using the connection.""" return SteadyDBCursor(self, *args, **kwargs) def __del__(self): """Delete the steady connection.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self._close() # make sure the connection is closed except: # noqa: E722, S110 pass class SteadyDBCursor: """A hardened version of DB-API 2 cursors.""" def __init__(self, con, *args, **kwargs): """Create a "tough" DB-API 2 cursor.""" # basic initialization to make finalizer work self._cursor = None self._closed = True # nesting level of no_failover() contexts self._no_failover = 0 # proper initialization of the cursor self._con = con self._args, self._kwargs = args, kwargs self._clearsizes() try: self._cursor = con._cursor(*args, **kwargs) except AttributeError as error: raise TypeError(f"{con!r} is not a SteadyDBConnection.") from error self._closed = False def __enter__(self): """Enter the runtime context for the cursor object.""" return self def __exit__(self, *exc): """Exit the runtime context for the cursor object.""" self.close() def __iter__(self): """Make cursor compatible to the iteration protocol.""" cursor = self._cursor try: # use iterator provided by original cursor return iter(cursor) except TypeError: # create iterator if not provided return iter(cursor.fetchone, None) @property def dbapi_cursor(self): """Return the underlying DB-API 2 cursor. Note that operations executed directly on this cursor are not covered by the failover mechanism and are not counted towards the maximum usage limit of the underlying steady connection. Consider using no_failover() instead, which keeps the bookkeeping of the steady connection intact. """ cursor = self._cursor if not cursor: raise InvalidCursorError return cursor @contextmanager def no_failover(self): """Suspend the failover mechanism inside this context. Statements that are executed on this cursor inside the context are not retried when they fail, but the error is raised to the application immediately. Everything else, particularly the ping check and the usage bookkeeping of the underlying steady connection, is left untouched. This is useful for statements that are expected to fail in a controlled way, e.g. when a deliberate server side timeout has been requested, where a retry would multiply the runtime. """ self._no_failover += 1 try: yield self finally: self._no_failover -= 1 def setinputsizes(self, sizes): """Store input sizes in case cursor needs to be reopened.""" self._inputsizes = sizes def setoutputsize(self, size, column=None): """Store output sizes in case cursor needs to be reopened.""" self._outputsizes[column] = size def _clearsizes(self): """Clear stored input and output sizes.""" self._inputsizes = [] self._outputsizes = {} def _setsizes(self, cursor=None): """Set stored input and output sizes for cursor execution.""" if cursor is None: cursor = self._cursor if self._inputsizes: cursor.setinputsizes(self._inputsizes) for column, size in self._outputsizes.items(): if column is None: cursor.setoutputsize(size) else: cursor.setoutputsize(size, column) def close(self): """Close the tough cursor. It will not complain if you close it more than once. """ if not self._closed: with suppress(Exception): self._cursor.close() self._closed = True def _get_tough_method(self, name): """Return a "tough" version of the given cursor method.""" def tough_method(*args, **kwargs): execute = name.startswith('execute') con = self._con transaction = con._transaction if not transaction: con._ping_check(4) overused = False try: # check whether the connection has been used too often if (con._maxusage and con._usage >= con._maxusage and not transaction): overused = True raise con._failure if execute: self._setsizes() method = getattr(self._cursor, name) result = method(*args, **kwargs) # try to execute if execute: self._clearsizes() except con._failures as error: # execution error # the connection must always be reset when it is overused, # otherwise the application can veto the failover mechanism if not overused and ( self._no_failover or not con._fatal_check(error)): raise if not transaction: try: cursor2 = con._cursor( *self._args, **self._kwargs) # open new cursor except Exception: # noqa: S110 pass else: try: # and try one more time to execute if execute: self._setsizes(cursor2) method = getattr(cursor2, name) result = method(*args, **kwargs) if execute: self._clearsizes() except Exception: # noqa: S110 pass else: self.close() self._cursor = cursor2 con._usage += 1 return result with suppress(Exception): cursor2.close() try: # try to reopen the connection con2 = con._create() except Exception: # noqa: S110 pass else: try: cursor2 = con2.cursor( *self._args, **self._kwargs) # open new cursor except Exception: # noqa: S110 pass else: if transaction: self.close() con._close() con._store(con2) self._cursor = cursor2 raise error # raise the original error again error2 = None try: # try one more time to execute if execute: self._setsizes(cursor2) method2 = getattr(cursor2, name) # if the following call hangs, # you may have forgotten to call begin() result = method2(*args, **kwargs) if execute: self._clearsizes() except error.__class__: # same execution error use2 = False error2 = error except Exception as error: # other execution errors use2 = True error2 = error else: use2 = True if use2: self.close() con._close() con._store(con2) self._cursor = cursor2 con._usage += 1 if error2: raise error2 # raise the other error return result with suppress(Exception): cursor2.close() with suppress(Exception): con2.close() if transaction: con._transaction = False raise error # re-raise the original error again else: con._usage += 1 return result return tough_method def __getattr__(self, name): """Inherit methods and attributes of underlying cursor.""" if self._cursor: if name.startswith(('execute', 'call')): # make execution methods "tough" return self._get_tough_method(name) return getattr(self._cursor, name) raise InvalidCursorError def __del__(self): """Delete the steady cursor.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self.close() # make sure the cursor is closed except: # noqa: E722, S110 pass WebwareForPython-DBUtils-8de754c/dbutils/steady_pg.py000066400000000000000000000256761524214146500227330ustar00rootroot00000000000000"""SteadyPg - hardened classic PyGreSQL connections. Implements steady connections to a PostgreSQL database using the classic (not DB-API 2 compliant) PyGreSQL API. The connections are transparently reopened when they are closed or the database connection has been lost or when they are used more often than an optional usage limit. Only connections which have been marked as being in a database transaction with a begin() call will not be silently replaced. A typical situation where database connections are lost is when the database server or an intervening firewall is shutdown and restarted for maintenance reasons. In such a case, all database connections would become unusable, even though the database service may be already available again. The "hardened" connections provided by this module will make the database connections immediately available again. This results in a steady PostgreSQL connection that can be used by PooledPg or PersistentPg to create pooled or persistent connections to a PostgreSQL database in a threaded environment such as the application server of "Webware for Python." Note, however, that the connections themselves are not thread-safe. For more information on PostgreSQL, see: https://www.postgresql.org/ For more information on PyGreSQL, see: http://www.pygresql.org For more information on Webware for Python, see: https://webwareforpython.github.io/w4py/ Usage: You can use the class SteadyPgConnection in the same way as you would use the class DB from the classic PyGreSQL API module db. The only difference is that you may specify a usage limit as the first parameter when you open a connection (set it to None if you prefer unlimited usage), and an optional list of commands that may serve to prepare the session as the second parameter, and you can specify whether is is allowed to close the connection (by default this is true). When the connection to the PostgreSQL database is lost or has been used too often, it will be automatically reset, without further notice. from dbutils.steady_pg import SteadyPgConnection db = SteadyPgConnection(10000, ["set datestyle to german"], host=..., dbname=..., user=..., ...) ... result = db.query('...') ... db.close() Ideas for improvement: * Alternatively to the maximum number of uses, implement a maximum time to live for connections. * Optionally log usage and loss of connection. Copyright, credits and license: * Contributed as supplement for Webware for Python and PyGreSQL by Christoph Zwerschke in September 2005 Licensed under the MIT license. """ from contextlib import suppress from pg import DB as PgConnection # noqa: N811 from . import __version__ __all__ = [ 'SteadyPgConnection', 'SteadyPgError', 'InvalidConnectionError', ] class SteadyPgError(Exception): """General SteadyPg error.""" class InvalidConnectionError(SteadyPgError): """Database connection is invalid.""" # deprecated alias names for error classes InvalidConnection = InvalidConnectionError class SteadyPgConnection: """Class representing steady connections to a PostgreSQL database. Underlying the connection is a classic PyGreSQL pg API database connection which is reset if the connection is lost or used too often. Thus the resulting connection is steadier ("tough and self-healing"). If you want the connection to be persistent in a threaded environment, then you should not deal with this class directly, but use either the PooledPg module or the PersistentPg module to get the connections. """ version = __version__ def __init__( self, maxusage=None, setsession=None, closeable=True, *args, **kwargs): """Create a "tough" PostgreSQL connection. A hardened version of the DB wrapper class of PyGreSQL. maxusage: maximum usage limit for the underlying PyGreSQL connection (number of uses, 0 or None means unlimited usage) When this limit is reached, the connection is automatically reset. setsession: optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to ...", "set time zone ..."] closeable: if this is set to false, then closing the connection will be silently ignored, but by default the connection can be closed args, kwargs: the parameters that shall be used to establish the PostgreSQL connections with PyGreSQL using pg.DB() """ # basic initialization to make finalizer work self._con = None self._closed = True # proper initialization of the connection if maxusage is None: maxusage = 0 if not isinstance(maxusage, int): raise TypeError("'maxusage' must be an integer value.") self._maxusage = maxusage self._setsession_sql = setsession self._closeable = closeable self._con = PgConnection(*args, **kwargs) self._transaction = False self._closed = False self._setsession() self._usage = 0 def __enter__(self): """Enter the runtime context. This will start a transaction.""" self.begin() return self def __exit__(self, *exc): """Exit the runtime context. This will end the transaction.""" if exc[0] is None and exc[1] is None and exc[2] is None: self.commit() else: self.rollback() def _setsession(self): """Execute the SQL commands for session preparation.""" if self._setsession_sql: for sql in self._setsession_sql: self._con.query(sql) def _close(self): """Close the tough connection. You can always close a tough connection with this method, and it will not complain if you close it more than once. """ if not self._closed: with suppress(Exception): self._con.close() self._transaction = False self._closed = True def close(self): """Close the tough connection. You are allowed to close a tough connection by default, and it will not complain if you close it more than once. You can disallow closing connections by setting the closeable parameter to something false. In this case, closing tough connections will be silently ignored. """ if self._closeable: self._close() elif self._transaction: self.reset() def reopen(self): """Reopen the tough connection. It will not complain if the connection cannot be reopened. """ try: self._con.reopen() except Exception: if self._transaction: self._transaction = False with suppress(Exception): self._con.query('rollback') else: self._transaction = False self._closed = False self._setsession() self._usage = 0 def reset(self): """Reset the tough connection. If a reset is not possible, tries to reopen the connection. It will not complain if the connection is already closed. """ try: self._con.reset() self._transaction = False self._setsession() self._usage = 0 except Exception: try: self.reopen() except Exception: with suppress(Exception): self.rollback() def begin(self, sql=None): """Begin a transaction.""" self._transaction = True try: begin = self._con.begin except AttributeError: return self._con.query(sql or 'begin') else: # use existing method if available return begin(sql=sql) if sql else begin() def end(self, sql=None): """Commit the current transaction.""" self._transaction = False try: end = self._con.end except AttributeError: return self._con.query(sql or 'end') else: return end(sql=sql) if sql else end() def commit(self, sql=None): """Commit the current transaction.""" self._transaction = False try: commit = self._con.commit except AttributeError: return self._con.query(sql or 'commit') else: return commit(sql=sql) if sql else commit() def rollback(self, sql=None): """Rollback the current transaction.""" self._transaction = False try: rollback = self._con.rollback except AttributeError: return self._con.query(sql or 'rollback') else: return rollback(sql=sql) if sql else rollback() def _get_tough_method(self, method): """Return a "tough" version of a connection class method. The tough version checks whether the connection is bad (lost) and automatically and transparently tries to reset the connection if this is the case (for instance, the database has been restarted). """ def tough_method(*args, **kwargs): transaction = self._transaction if not transaction: try: # check whether connection status is bad # or the connection has been used too often if not self._con.db.status or ( self._maxusage and self._usage >= self._maxusage): raise AttributeError except Exception: self.reset() # then reset the connection try: result = method(*args, **kwargs) # try connection method except Exception: # error in query if transaction: # inside a transaction self._transaction = False raise # propagate the error if self._con.db.status: # if it was not a connection problem raise # then propagate the error self.reset() # reset the connection result = method(*args, **kwargs) # and try one more time self._usage += 1 return result return tough_method def __getattr__(self, name): """Inherit the members of the standard connection class. Some methods are made "tougher" than in the standard version. """ if self._con: attr = getattr(self._con, name) if (name in ('query', 'get', 'insert', 'update', 'delete') or name.startswith('get_')): attr = self._get_tough_method(attr) return attr raise InvalidConnectionError def __del__(self): """Delete the steady connection.""" # builtins (including Exceptions) might not exist anymore try: # noqa: SIM105 self._close() # make sure the connection is closed except: # noqa: E722, S110 pass WebwareForPython-DBUtils-8de754c/docs/000077500000000000000000000000001524214146500176445ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/docs/changelog.html000066400000000000000000000365411524214146500224720ustar00rootroot00000000000000 Changelog for DBUtils

Changelog for DBUtils

3.2.0

DBUtils 3.2.0 was released on August 21, 2026.

Changes:

Bugfixes:

3.1.2

DBUtils 3.1.2 was released on September 7, 2025.

Changes:

3.1.1

DBUtils 3.1.1 was released on June 4, 2025.

Changes:

3.1.0

DBUtils 3.1.0 was released on March 17, 2024.

Changes:

3.0.3

DBUtils 3.0.3 was released on April 27, 2023.

Changes:

3.0.2

DBUtils 3.0.2 was released on January 14, 2022.

The optional iterator protocol on cursors is now supported.

3.0.1

DBUtils 3.0.1 was released on December 22, 2021.

It includes InterfaceError to the default list of exceptions for which the connection failover mechanism is applied. You can override this with the failures parameter.

3.0.0

DBUtils 3.0.0 was released on November 26, 2021.

It is intended to be used with Python versions 3.6 to 3.10.

Changes:

2.0.3

DBUtils 2.0.3 was released on November 26, 2021.

Changes:

2.0.2

DBUtils 2.0.2 was released on June 8, 2021.

Changes:

2.0.1

DBUtils 2.0.1 was released on April 8, 2021.

Changes:

2.0

DBUtils 2.0 was released on September 26, 2020.

It is intended to be used with Python versions 2.7 and 3.5 to 3.9.

Changes:

1.4

DBUtils 1.4 was released on September 26, 2020.

It is intended to be used with Python versions 2.7 and 3.5 to 3.9.

Improvements:

1.3

DBUtils 1.3 was released on March 3, 2018.

It is intended to be used with Python versions 2.6, 2.7 and 3.4 to 3.7.

Improvements:

1.2

DBUtils 1.2 was released on February 5, 2017.

It is intended to be used with Python versions 2.6, 2.7 and 3.0 to 3.6.

1.1.1

DBUtils 1.1.1 was released on February 4, 2017.

It is intended to be used with Python versions 2.3 to 2.7.

Improvements:

Bugfixes:

1.1

DBUtils 1.1 was released on August 14, 2011.

Improvements:

Bugfixes:

1.0

DBUtils 1.0 was released on November 29, 2008.

It is intended to be used with Python versions 2.2 to 2.6.

Changes:

Bugfixes and improvements:

0.9.4

DBUtils 0.9.4 was released on July 7, 2007.

This release fixes a problem in the destructor code and has been supplemented with a German User's Guide.

Again, please note that the dbapi parameter has been renamed to creator in the last release, since you can now pass custom creator functions for database connections instead of DB-API 2 modules.

0.9.3

DBUtils 0.9.3 was released on May 21, 2007.

Changes:

0.9.2

DBUtils 0.9.2 was released on September 22, 2006.

It is intended to be used with Python versions 2.2 to 2.5.

Changes:

0.9.1

DBUtils 0.9.1 was released on May 8, 2006.

It is intended to be used with Python versions 2.2 to 2.4.

Changes:

0.8.1 - 2005-09-13

DBUtils 0.8.1 was released on September 13, 2005.

It is intended to be used with Python versions 2.0 to 2.4.

This is the first public release of DBUtils.

WebwareForPython-DBUtils-8de754c/docs/changelog.rst000066400000000000000000000247121524214146500223330ustar00rootroot00000000000000Changelog for DBUtils +++++++++++++++++++++ 3.2.0 ===== DBUtils 3.2.0 was released on August 21, 2026. Changes: * Added an ``isfatal`` parameter to ``PersistentDB``, ``PooledDB`` and ``steady_db.connect()``, an optional callable that decides whether a given exception is really fatal for the connection and the failover mechanism shall be applied. This allows distinguishing errors that merely report a failed statement, such as a deliberate server side statement timeout, from errors that report a broken connection, when the database module maps both onto the same exception class. * The ``ping`` parameter of ``PersistentDB``, ``PooledDB`` and ``steady_db.connect()`` now also accepts an SQL statement such as ``"select 1"`` or a callable that shall be used for checking the connections instead of the ``ping()`` method, which is not part of the DB-API 2 specification and not provided by every database module. Pass a tuple with the usual integer value and the SQL statement or the callable if the connections shall not only be checked when they are requested. * Added a ``no_failover()`` context manager to steady cursors that suspends the failover mechanism for individual statements. * Added ``dbapi_connection`` and ``dbapi_cursor`` attributes providing supported access to the underlying DB-API 2 objects. Bugfixes: * Fixed the transaction flag being reset on the cursor instead of the connection when a connection could not be reopened during a transaction, which kept the failover mechanism suspended. * Fixed an ``IndexError`` in ``PooledDB.connection()`` when a thread waiting for a shared connection that was in a transaction got woken up after that connection had become idle and left the shared cache. * Fixed ``PooledPgConnection.reopen()`` wrapping another pooled connection instead of the underlying steady connection when it was called after the connection had already been returned to the pool. * Fixed ``pooled_db.SharedDBConnection`` violating the contract between ``__eq__`` and ``__hash__``; shared connections now compare by identity. * Fixed ``PooledPg.close()`` releasing the semaphore for the connections in the pool cache, which had already released it when they were returned to the pool, so that the pool handed out more connections at the same time than allowed by ``maxconnections`` after it had been closed. * Fixed ``PooledPg`` not releasing the semaphore when a connection could not be established, so that every failed attempt permanently reduced the number of connections the pool was willing to hand out. 3.1.2 ===== DBUtils 3.1.2 was released on September 7, 2025. Changes: * Support Python version 3.14. 3.1.1 ===== DBUtils 3.1.1 was released on June 4, 2025. Changes: * Support Python version 3.13. 3.1.0 ===== DBUtils 3.1.0 was released on March 17, 2024. Changes: * Support Python version 3.12, cease support for Python 3.6. * Various small internal improvements and modernizations. 3.0.3 ===== DBUtils 3.0.3 was released on April 27, 2023. Changes: * Support Python version 3.11. * Improve determination of DB API module if creator is specified. * Minor fixes and section an advanced usage in docs. 3.0.2 ===== DBUtils 3.0.2 was released on January 14, 2022. The optional iterator protocol on cursors is now supported. 3.0.1 ===== DBUtils 3.0.1 was released on December 22, 2021. It includes ``InterfaceError`` to the default list of exceptions for which the connection failover mechanism is applied. You can override this with the ``failures`` parameter. 3.0.0 ===== DBUtils 3.0.0 was released on November 26, 2021. It is intended to be used with Python versions 3.6 to 3.10. Changes: * Cease support for Python 2 and 3.5, minor optimizations. 2.0.3 ===== DBUtils 2.0.3 was released on November 26, 2021. Changes: * Support Python version 3.10. 2.0.2 ===== DBUtils 2.0.2 was released on June 8, 2021. Changes: * Allow using context managers for pooled connections. 2.0.1 ===== DBUtils 2.0.1 was released on April 8, 2021. Changes: * Avoid "name Exception is not defined" when exiting. 2.0 === DBUtils 2.0 was released on September 26, 2020. It is intended to be used with Python versions 2.7 and 3.5 to 3.9. Changes: * DBUtils does not act as a Webware plugin anymore, it is now just an ordinary Python package (of course it could be used as such also before). * The Webware ``Examples`` folder has been removed. * Folders, packages and modules have been renamed to lower-case. Particularly, you need to import ``dbutils`` instead of ``DBUtils`` now. * The internal naming conventions have also been changed to comply with PEP8. * The documentation has been adapted to reflect the changes in this version. * This changelog has been compiled from the former release notes. 1.4 === DBUtils 1.4 was released on September 26, 2020. It is intended to be used with Python versions 2.7 and 3.5 to 3.9. Improvements: * The ``SteadyDB`` and ``SteadyPg`` classes only reconnect after the ``maxusage`` limit has been reached when the connection is not currently inside a transaction. 1.3 === DBUtils 1.3 was released on March 3, 2018. It is intended to be used with Python versions 2.6, 2.7 and 3.4 to 3.7. Improvements: * This version now supports context handlers for connections and cursors. 1.2 === DBUtils 1.2 was released on February 5, 2017. It is intended to be used with Python versions 2.6, 2.7 and 3.0 to 3.6. 1.1.1 ===== DBUtils 1.1.1 was released on February 4, 2017. It is intended to be used with Python versions 2.3 to 2.7. Improvements: * Reopen ``SteadyDB`` connections when commit or rollback fails (suggested by Ben Hoyt). Bugfixes: * Fixed a problem when running under Jython (reported by Vitaly Kruglikov). 1.1 === DBUtils 1.1 was released on August 14, 2011. Improvements: * The transparent reopening of connections is actually an undesired behavior if it happens during database transactions. In these cases, the transaction should fail and the error be reported back to the application instead of the rest of the transaction being executed in a new connection and therefore in a new transaction. Therefore DBUtils now allows suspending the transparent reopening during transactions. All you need to do is indicate the beginning of a transaction by calling the ``begin()`` method of the connection. DBUtils makes sure that this method always exists, even if the database driver does not support it. * If the database driver supports a ``ping()`` method, then DBUtils can use it to check whether connections are alive instead of just trying to use the connection and reestablishing it in case it was dead. Since these checks are done at the expense of some performance, you have exact control when these are executed via the new ``ping`` parameter. * ``PooledDB`` has got another new parameter ``reset`` for controlling how connections are reset before being put back into the pool. Bugfixes: * Fixed propagation of error messages when the connection was lost. * Fixed an issue with the ``setoutputsize()`` cursor method. * Fixed some minor issues with the ``DBUtilsExample`` for Webware. 1.0 === DBUtils 1.0 was released on November 29, 2008. It is intended to be used with Python versions 2.2 to 2.6. Changes: * Added a ``failures`` parameter for configuring the exception classes for which the failover mechanisms is applied (as suggested by Matthew Harriger). * Added a ``closeable`` parameter for configuring whether connections can be closed (otherwise closing connections will be silently ignored). * It is now possible to override defaults via the ``creator.dbapi`` and ``creator.threadsafety`` attributes. * Added an alias method ``dedicated_connection`` as a shorthand for ``connection(shareable=False)``. * Added a version attribute to all exported classes. * Where the value ``0`` has the meaning "unlimited", parameters can now be also set to the value ``None`` instead. * It turned out that ``threading.local`` does not work properly with ``mod_wsgi``, so we use the Python implementation for thread-local data even when a faster ``threading.local`` implementation is available. A new parameter ``threadlocal`` allows you to pass an arbitrary class such as ``threading.local`` if you know it works in your environment. Bugfixes and improvements: * In some cases, when instance initialization failed or referenced objects were already destroyed, finalizers could throw exceptions or create infinite recursion (problem reported by Gregory Pinero and Jehiah Czebotar). * DBUtils now tries harder to find the underlying DB-API 2 module if only a connection creator function is specified. This had not worked before with the MySQLdb module (problem reported by Gregory Pinero). 0.9.4 ===== DBUtils 0.9.4 was released on July 7, 2007. This release fixes a problem in the destructor code and has been supplemented with a German User's Guide. Again, please note that the ``dbapi`` parameter has been renamed to ``creator`` in the last release, since you can now pass custom creator functions for database connections instead of DB-API 2 modules. 0.9.3 ===== DBUtils 0.9.3 was released on May 21, 2007. Changes: * Support custom creator functions for database connections. These can now be used as the first parameter instead of an DB-API module (suggested by Ezio Vernacotola). * Added destructor for steady connections. * Use setuptools_ if available. * Some code cleanup. * Some fixes in the documentation. Added Chinese translation of the User's Guide, kindly contributed by gashero. .. _setuptools: https://github.com/pypa/setuptools 0.9.2 ===== DBUtils 0.9.2 was released on September 22, 2006. It is intended to be used with Python versions 2.2 to 2.5. Changes: * Renamed ``SolidDB`` to ``SteadyDB`` to avoid confusion with the "solidDB" storage engine. Accordingly, renamed ``SolidPg`` to ``SteadyPg``. 0.9.1 ===== DBUtils 0.9.1 was released on May 8, 2006. It is intended to be used with Python versions 2.2 to 2.4. Changes: * Added ``_closeable`` attribute and made persistent connections not closeable by default. This allows ``PersistentDB`` to be used in the same way as you would use ``PooledDB``. * Allowed arguments in the DB-API 2 ``cursor()`` method. MySQLdb is using this to specify cursor classes. (Suggested by Michael Palmer.) * Improved the documentation and added a User's Guide. 0.8.1 - 2005-09-13 ================== DBUtils 0.8.1 was released on September 13, 2005. It is intended to be used with Python versions 2.0 to 2.4. This is the first public release of DBUtils. WebwareForPython-DBUtils-8de754c/docs/dependencies_db.png000066400000000000000000000106751524214146500234560ustar00rootroot00000000000000‰PNG  IHDRr &{«»PLTE@€ Àÿ@@@@€@ @À@ÿ€€@€€€ €À€ÿ  @ €   À ÿÀÀ@À€À ÀÀÀÿÿÿ@ÿ€ÿ ÿÀÿÿ@@@@€@ @À@ÿ@@@@@@@€@@ @@À@@ÿ@€@€@@€€@€ @€À@€ÿ@ @ @@ €@  @ À@ ÿ@À@À@@À€@À @ÀÀ@Àÿ@ÿ@ÿ@@ÿ€@ÿ @ÿÀ@ÿÿ€€@€€€ €À€ÿ€@€@@€@€€@ €@À€@ÿ€€€€@€€€€€ €€À€€ÿ€ € @€ €€  € À€ ÿ€À€À@€À€€À €ÀÀ€Àÿ€ÿ€ÿ@€ÿ€€ÿ €ÿÀ€ÿÿ  @ €   À ÿ @ @@ @€ @  @À @ÿ € €@ €€ €  €À €ÿ    @  €     À  ÿ À À@ À€ À  ÀÀ Àÿ ÿ ÿ@ ÿ€ ÿ  ÿÀ ÿÿÀÀ@À€À ÀÀÀÿÀ@À@@À@€À@ À@ÀÀ@ÿÀ€À€@À€€À€ À€ÀÀ€ÿÀ À @À €À  À ÀÀ ÿÀÀÀÀ@ÀÀ€ÀÀ ÀÀÀÀÀÿÀÿÀÿ@Àÿ€Àÿ ÀÿÀÀÿÿÿÿ@ÿ€ÿ ÿÀÿÿÿ@ÿ@@ÿ@€ÿ@ ÿ@Àÿ@ÿÿ€ÿ€@ÿ€€ÿ€ ÿ€Àÿ€ÿÿ ÿ @ÿ €ÿ  ÿ Àÿ ÿÿÀÿÀ@ÿÀ€ÿÀ ÿÀÀÿÀÿÿÿÿÿ@ÿÿ€ÿÿ ÿÿÀÿÿÿ 000PPP```pppxxxˆˆˆ˜˜˜¨¨¨°°°¸¸¸ÈÈÈÐÐÐØØØàààÝÏLebKGDˆH pHYs  šœ›IDATxÚíË–¢¼€™Õ‹œQ‡QYïÁHý§Õœ5²ÑQ=iŽ|GDÀþ ×â’Ç8¹pU°C%Y« EÈN¾lvîApPòùäò! È{ˆÜ²|À!ø–uY2 @Á·¾Î4§,^ ]Fn—\uwÀ<]p!÷ÈwI©pœS¯ ¸p]Õ#Ç |ÄëºY €ÕÒsKÄ6LUKÁ*gV9ó”Uße*÷g±*ï,^Jù-"G€; Ê³V«9Ä\ëtøPFˆ r• þ²ª²»–Àž¶¤fŠåÆÈ g9fþ¯M³\ôEÆ…ÛP¼çz%DÓª3#ɬPEËtOùJ1Ê!,ëlTnsFøW+ªUw¢¦é\¸£*gé( ®ÌMùdh]•G^42M+ ¿Fùþ¶Å¯¿Ÿ6ůž6¶ûrªY'9®RrÝË>®¼sàÚ‹O"¶^S?pÛ¤•û Ó#)×ØUƒÉ!à~Ázѯ$çøÎ궾µ–m*%‰‘"_;Ó`‚c%†…~Äõ§c Ý™­<#Óòhÿf}Y–ûDp}#1‹÷TËYj yäJ£¢‘Xì'à’Ÿ•ÔѸqDù‘4$pmLZñLæy+‰~õ¶¶7Ó;k{>¹œL ‹qþaÍ ÕÈZ,äc„ÛD>F`â–Å…Ï%ÁøAð&EÁ•®Dt.‚ˆ´=ÏÛf©)äñFº@£¢+T|iÉ·ÕáÏäp”rª;\—úõ©á’Ϊq̇ڷ«ÂŽÍ6yÛ뇵29×K¹YpÍÐÙ¥a­óÔ¤"b4*‡µŠ i];΀’ù¦[†—29Öƒ¸—×õ3/õ?ÏâJE 4ªo…MQ´?zÔisצ(„MÄÅ ÝéÇ ó|›ˆ7ÝÜâlóW‘wlºŠ[Tá» h%çÖ—H»µ¨‚Õ"×Õ(ÎõK‡OrÎ?¸W]¼wÄ¡¦kÇ- -ÄŒ¹Õgä×KYl;jmîȃ†é2zû±ëˆm ¶‚¼©eºA1ðc ¶îlyC qÕ†þÄÉ?õy3xCŽ°ïÈ›U âgÞcY‡Z@ÞHa…›riÄÛ@Þ¤E®õ(Ä[qÒ 7¢;B‰·‚¼A*¸ö”I¼8·wkÄ"J¼ä·7Í…Öžr‰·ƒüæÊPhí)™xK¾µnÍ”ÈÚS6ñ–ßZ F”N¼-r7V â†åo ùj+¬öìñ¶ßfY„ #vxk®)oÊ›¨Ú³Ä[C~SwRPíÙ â\UÊ;·¬/-¦Úšæïq~Èym9ßèñ͈óBÎuÓ“ÕÆæ­Îç„œ÷Ö¾¼…v†8äü7°r×Éj·¤ýEÞB‹ƒ³IÄð;!oÅÇ&×bìq.È+@åsUW‹Vtþ ÏñÜN缪»~ê;2 'ž“@|ÿ,Ï~á§æÝ"Îy›Sß‘)òÊÁ”jäÜF:Fœò’:Æc^!ó™xm4=‰·F'Ö‰7$=sÓD|NÎ)òŸ¾ñ‹‡ç;ç¼4Ø­=sðF}GÆ?Ô@hçÄ9óÖø´1±:ÇÄß «ý4ÐŒ\Qƒ÷ý¬¯õ-‰sA^òi‰;ýy­Ëý;Æš»eÞ™?Ãeî·US·0"Z¾7N]-qAÞ=â\,&Øï±!¶üóÚˆÞÁ¿¹Ã¼5B m”ñFKØr ðn²t8ä¥qÚ0ŠLG¥Zžxm\:¿ˆBæ­Ÿ Iî@SFp÷Ÿz*yjãQ}v‘8ï ò@`þáFÈs˜xm\;úñ´eΟ6¬Rð\¹Ü[‚É~QW’߈8ä% €ÆÌÏ4ñɼ6"ŒŽàWâBrC(kÉ?µàwÝ|Ç¡ºIœKïóÆNËûU{sîo#v”8ä7 $†`?Ňßl hÆ%Î^ç3’x˸Vâç1bs@§'§GÄ9MQð~ºXw‰óšˆãë|Ðߘ8·é渾ÅCÕyä·µã<‘SÕt¹ø9¼ß;k§‰ó]:„¾vêL¾ZxñMûœ-!ï‰ïÞë$rä´ít³; V:‚œ¸Ñòš¸xäÄe‹Æ¥ûÄÅ#§~æZ[Âßâ[-úëq È–m—^—ŸÓì›P`æ%èDËubgÂäpþo0ÁÕçò9øÄÕgŒãÚrl„‚C><ùØëX Ï8ÝGA)v<$l,„R­fg ¾*­Â_Ä"ˆ¬(Æ×Çô@îÄqÇ•Öì‘Kð?øšD *×Ý[‚#Ñ:ô~OIÈï%éôÎa{…¸MS¨Ûû³nBäõó¯ö‚v}¢Ð­¶ô•ïm>/ ø.ȇ0 ȇ0 ¡Qø?äå n&jIEND®B`‚WebwareForPython-DBUtils-8de754c/docs/dependencies_pg.png000066400000000000000000000103211524214146500234630ustar00rootroot00000000000000‰PNG  IHDRr &{«»PLTE@€ Àÿ@@@@€@ @À@ÿ€€@€€€ €À€ÿ  @ €   À ÿÀÀ@À€À ÀÀÀÿÿÿ@ÿ€ÿ ÿÀÿÿ@@@@€@ @À@ÿ@@@@@@@€@@ @@À@@ÿ@€@€@@€€@€ @€À@€ÿ@ @ @@ €@  @ À@ ÿ@À@À@@À€@À @ÀÀ@Àÿ@ÿ@ÿ@@ÿ€@ÿ @ÿÀ@ÿÿ€€@€€€ €À€ÿ€@€@@€@€€@ €@À€@ÿ€€€€@€€€€€ €€À€€ÿ€ € @€ €€  € À€ ÿ€À€À@€À€€À €ÀÀ€Àÿ€ÿ€ÿ@€ÿ€€ÿ €ÿÀ€ÿÿ  @ €   À ÿ @ @@ @€ @  @À @ÿ € €@ €€ €  €À €ÿ    @  €     À  ÿ À À@ À€ À  ÀÀ Àÿ ÿ ÿ@ ÿ€ ÿ  ÿÀ ÿÿÀÀ@À€À ÀÀÀÿÀ@À@@À@€À@ À@ÀÀ@ÿÀ€À€@À€€À€ À€ÀÀ€ÿÀ À @À €À  À ÀÀ ÿÀÀÀÀ@ÀÀ€ÀÀ ÀÀÀÀÀÿÀÿÀÿ@Àÿ€Àÿ ÀÿÀÀÿÿÿÿ@ÿ€ÿ ÿÀÿÿÿ@ÿ@@ÿ@€ÿ@ ÿ@Àÿ@ÿÿ€ÿ€@ÿ€€ÿ€ ÿ€Àÿ€ÿÿ ÿ @ÿ €ÿ  ÿ Àÿ ÿÿÀÿÀ@ÿÀ€ÿÀ ÿÀÀÿÀÿÿÿÿÿ@ÿÿ€ÿÿ ÿÿÀÿÿÿ 000PPP```pppxxxˆˆˆ˜˜˜¨¨¨°°°¸¸¸ÈÈÈÐÐÐØØØàààÝÏLebKGDˆH pHYs  šœ ¯IDATxÚí=~£:‡é²‘·¥Š³W3y§ÊnEªL*g™Ên‚\e!ãÊ2à™ß¤eÜ# Œ° ±‰>p…?À€Î£ÃÑ{dX,/Þ€`@> –ù€|X䧈œa- ë±ÕšL‚,¼9¶¦•S3i?/kÎW(¨6j¤±uµ 9J«s¬ …Bæ,Óê×ë]ˆø¦»5ô·õºÝƒ¯Þ¨ÎŸî Vγ-Xvó¯`ºÂC~ÌÆá”F2ÌÀ4Ân×£¬á j]7…/yjŠ¢( [–!/õK­G•ŠGÒ8k¦QÔDé^£[¬ÆÐÄùá‚F#Âüˆ•ï¼^ƒõ5™…ùJÍÆÛz-÷à«é7õ Û XÕy`Ÿ¢`¢Ø`4Ê cô\ ñ×Ê–ðrù§uŒÂh±Œ¼ð¢Íº†š®N= 3áÖ>G~ÎØ·ð£çÒÅF½4ÞAŽÎùÕ; ÖÓÚ;¯×Ùú:½[߈U³ê$åð#òxvr$ „Pl öoyrå#4"OÞáFfëÒ`«ü¬±i’§þ ®=¡…—_Ê’û€¼°@±±É6å}F´hâWÒh@ž¨^»ÀïÅÁ•w^¯p *³ru_õò¤ÅËeÁŠbsh$(xg&ÞÈ¢ó4#ðò)\ u䵺Û‘K³\*Tzù¥,G^X Ø¨•Æ¶A÷cî ÁUu+.%¸|ζï¼^¹Oæ”Pq))õZì!.%ÕËÕ6b×ËeÁŠbsû  ò\;{ᜫo)¿óü'ÈÛ݃£•ºnj¥¥‰Aœ‹ºÛóribã±4ÎÚhàö$q5>Ch]6÷Õ;o£}ˆ¼ç¼Ýàë»{Àêx=¿:TWÓj óCZ[ Ÿ<À˼ÁMxÔäŽó´Î /¿&ÖU^¾[Ó•‰,QtÁö½¼°@±ñHat¹šÉBÍËÄi7-¹[&7åþZ£<àÝC Hä,ªšÏšugÏmIba"“C5OržÇq[* T5Òغš×tEÍxºŒ)ö ›ß.ZúšUÕÂY¶9a^ãQ~öÁK.’Dù3º%òF]7w¸Úú 6ïØö!U¼ÆÛ*¾8ðK9Ór¾]±¶¾æÖnE»EÖ!ÕºQû]…7n½¼aóžmhÄ%ª@^…öûM´ÁtV®[/º¹Åº–» ‰ý!¡½»§µQ!í#¡ö{ÇFvo7Ì“5n¢Ð>>†Qó0¤Éa¸·kÚˆ‰‡Ý¸­ÔÛNI˜¾½9êevò­šÖoâ¡å©]ZÞžƒD(Õ1Ÿ$EQË5L‘1gcïÕ§6[´SžýÙZð›®‹*_Ú'‹ø…þ¥½¦[M =3s‡v'h“½ã©“;O ¢?,ÒéDŽ=ï#¥~ %pEÀ ÙI"÷º7M°7vœ%s—Ƚ®y/EöŒnpqϳåäZ‘'^w朸‡Ü¹¸ÅúÖ‰y™ âömÏÅmV· ä¸ó®Ø•‹Û<µvä’´ÂlÁ—~BN9†žP·D§è¡ç ˜—.nõúÒ‰œ'ˆMã_‡E󺮖gi–.n·®u?—t-¼˜j÷™.¶%nµÑœu½Ždˆ¹hC´?÷Ùu^mâ")/˜ÛMNµ#ï:k¾s#ð¡å%´Ÿ(麹;¹¸Á’ šXïyÌ8•P.ŸÛ½ýÈ»ùkŠ'ä•:òn ¨ƒ§x\M°0€¼SjÜÝÆSb¶wi@í·žÔÙÓqž‘+¶ÿ­§;âFwˆ,¶;B‰›Q:ÞË!—ÄÍ ?>ã³Ûz:%nùÑ¡ågë7$gvlh¶Û² Û`ù±^kÕíœÍ17ŠüX†6C¹kâ¦)lv„œ7…ü8ˆ;Bî‰S=ªµ×zÒ“O…\Ù| Ó#æ„«?þ¼ ¹väiÑš„c{°¤ĵ!§(J´x(ãB$Ÿ˜¸.äzŸyÂFì qMÈu?ÙÇL$=!®¹ýwswóõ"güGû1{kDnd|AóÍÅþ×¼±§Ù,d8;X‚PsèíqÈ›ºëo*ÚWV»¤}"®yº£±Éej¤¡”?$¸”¨¡Ñ¢ö)ÒðBÁ‡¿áÜÌgó'L!¯E€»?Ñ !C)ÈÆÿ. ñÆp1ƒ½o¥jžÖ°|Âk,ôÌZãÕI'ä‘×GŒ¦¹PÕò‡á‚‹3ùÑó²ÐXËC¾eZ!Ÿœ P†d¶¾|븟ˆ¸äêèüxÅ)±¼Ôw¤éK= Ì\Ip|NüJà‰kÂq}8.Fµ M ïqÈk9"K7\@–{y!ø;JrŸ#Ϥ^æS”F×ÍÈÿw©?Kìqí‹?b\¥¬ ,9ž Èi© KuE@îÈR–œ¼z—Ú›OÖ;âZ«î8 …j#2”ò‡ìw4zå¢wa¡í{#¢xy¼m>7µX®§ƒ• ò ‘×S–\æ|BÈPª6æ8ãMèV¼‘‹}*ÈqV&‰\tRsÂÒ«„\#ò£Ò^–Ž¹²i.‡€(×÷WÇ=N„SÍîÙGâzî$ƒg‰bö—º~B9»…o÷š{/½$®¹þ6JKžÑO⚆(tÿ–Q¿~uóu#JyÚn¶b-Š~ýKÈõ"/þD3Õ!+ˆ´Ô^o‰kUª8Da±&¥Ø,+¨E;¡¿Ä­ÿwsòÃJ€í1qëÈYhc–mŸ‰Ûÿ‡òä#ZŠúÃ_9 » ž—³yÝÍ ;zO»@î³°«²ßç îy©ÐgÊÑûNÜré榽÷Ä] ß Qšpô><&ÑC䥛pô^'ä‘WnîiÎæN¸ä[7×wO‚¸äÛ¤…|AâŽK7¾R7ß5ráæ/šõ»›ï¹¸¡H?íò>"'¯Tsð=âÎënðN‡¸cäÚ˜Ÿq×È5Í|èëü‰^"×âŸ'’÷¹æ§E¼È?ÌüĈ÷ù™Ÿqqû8á¦yÕõslèèœr¬E'ç}„ È‹ %6<“Ñ95äØ¢óáÞ:ºMä‰UÏ£ѯŽÜz±‡OÃÙEn?™cýLí!w`?gXXCnY¢¼Ç¡ÅûÄNîêoÐz‚Ü‘ñ}Œæ¶wAÐÏñ‡Ì_QÄÿïp„ ÈCîOo××d¾¾JÇ5 £§ýÒÈ×SÂ¥FøSÿÙz@n9×T£&I¢¹)/¿!+à Ôáó€ÜJ,GÓd}OfëQ0ÄrK^þsŒ^ W=Ÿ«ú8rs±\jUfÐqý= ·Ö|Šü¡ðüy@nÞvV*(ЮÙrëË€|@> ÈMØÎ^!‰ãøj;bÁæðõ¾>€ÁWMůãø2ÅÕ¦~¸Õ¦)ùU蟳«¹bûÝyâ þÁE¹Éÿ î®jÄáûþ@ظSxYÅuÄ»ßw³Ëé€|kû `7Z¾Ͼ¥ó‰üYþù»aÁ&ž02ã¿ ”Ü ±Jºa â€û?ÿW!Ûð F¾Þær+aØ€¼´}}HáåàÈþ0¿ü7„%z€€Áø_¥|õF’/*äË c›gø¾ü Us ÛÈìºxõ3¹BÌb@^Ú~ >øA¤_®Èüžütp£Å2æá&Ås.(çqåÛÿ(ȃë*–³dDü«ê¥xÍ‚â®ÀÓÍ€¼´ûtÉc³?Ïä—Œ»ßK@üáÓh‰èj#X)ZØí!ŸÙm<¹Aõ;º.^+äåûWˆ+‡›³"–‹ ãÇ•ˆÔ2xpäÙ~š‘ï¼~€:û׉Ò\ò_d¼ 8Õùˆ×™|õs¹•YÑ~~åáæÒxîᥗ“<šŠöðÂÈ„päÌŸ¤ïIOp°y¦›ôßqćã¦Åm¶¹Ÿ‹ÄéÏ͈ÜNÒdR¼ú™ÜJÊâKOª(g$2È ÝÉ®±™gX~Ήœ÷ÏHñBPMÔ—ÿC„…=\Å¿‰×œ[Wä ¾öÔ¡Òü™òÿYÁÄйæSµš¿,òbRfÕ3$·Ssæ^Å+ýêÓ@ µü`C_ŸÛ²<¥ßâTPŒz*âÙõ;kÐq²üDœ.ay;²óŸùÂò]ôÈÎäÃ2 ȇe@~ú˹À =·f¸IEND®B`‚WebwareForPython-DBUtils-8de754c/docs/doc.css000066400000000000000000000021201524214146500211160ustar00rootroot00000000000000/* Style sheet for DBUtils documentation pages. */ /* First import default style for pages created with Docutils: */ @import url(docutils.css); /* Customization for DBUtils goes here: */ p { margin-top: 6pt; margin-bottom: 6pt; text-align: justify; } li { margin-bottom: 6pt; } h1, h2 { color: #002352; } h3, h4 { color: #002352; } h1 { font-size: 18pt; } h2 { font-size: 16pt; } h3 { font-size: 14pt; } h1.title { padding: 4pt; margin-bottom: 12pt; text-align: center; border-bottom: 1pt solid #025; padding-bottom: 8pt; } .contents ul { list-style: none; margin-bottom: 24pt; padding-left: 0em; margin-left: 2em; } .contents ul li { font-size: 14pt; margin-bottom: 2pt; } .contents ul ul { list-style-type: none; margin-top: 1pt; margin-bottom: 1pt; padding-left: 0em; margin-left: 1.5em; } .contents ul ul li { font-size: 13pt; margin-bottom: 1pt; } .contents > p.topic-title { font-size: 24pt; } .warning { color: brown; } .admonition-title { background-color: #F5F5DC; padding: 1pt 3pt; } .admonition-title::before { content: "âš "; margin-right: .5em; } WebwareForPython-DBUtils-8de754c/docs/docutils.css000066400000000000000000000141111524214146500222020ustar00rootroot00000000000000/* CSS 3 style sheet for the output of Docutils 0.21 HTML writer. */ div.dedication,nav.contents{padding:0;font-style:italic}h1.title,table tr{text-align:left}.footnote,pre.code,pre.doctest-block,pre.literal-block,pre.math{overflow:auto}body{font-family:Georgia,serif;background-color:#fafaf6;font-size:1.2em;line-height:1.4;margin:auto}main{counter-reset:figure table}footer,header,main{padding:.5em 5%;background-color:#fefef8;max-width:100rem}.citation,.footnote,.topic,div.line-block,dl,h1,h2,h3,h4,h5,h6,li,ol,p,table,ul{margin-top:.5em;margin-bottom:.5em}.topic,blockquote,figure{margin:.5em 2%;padding-left:1em}dl>dd{margin-bottom:.5em}p:first-child{margin-top:0}p:last-child{margin-bottom:0}div.line-block div.line-block,dl.option-list,figure>img,pre.code,pre.doctest-block,pre.literal-block,pre.math{margin-left:calc(2% + 1em)}footer,header{font-size:smaller}h2,h3,h4,p.section-subtitle,p.sidebar-subtitle,p.sidebar-title,p.subtitle,p.topic-title{font-weight:400;font-style:italic;text-align:left}.sectnum{font-style:normal}h1.title{margin-top:2.4em;margin-bottom:2em;font-size:2.4em}h1+p.subtitle{margin-top:-2em;margin-bottom:2em;font-size:2em}section{margin-top:2em}.contents>p.topic-title,h2{font-size:2.2em}h2+p.section-subtitle{font-size:1.6em}h3{font-size:1.2em}h3+p.section-subtitle{font-size:1.1em}figure.marginal>figcaption,h4,p.section-subtitle{font-size:1em}div.dedication{margin-left:0;font-size:1.2em}div.dedication p.topic-title{display:none}.topic p.attribution,blockquote p.attribution{text-align:right}ul.auto-toc>li>p{padding-left:1em;text-indent:-1em}nav.contents ul{padding-left:1em}hr{border:0;border-top:1px solid #ccc;margin:1em 10%}ol,ul{padding-left:1.1em}dd{margin-left:1.5em}dd>dl:first-child,dd>ol:first-child,dd>ul:first-child{clear:left}dl.docinfo>dd,dl.field-list>dd,dl.option-list>dd{margin-left:4em}dl.field-list.narrow>dd{margin-left:3em}dl.field-list.run-in>dd p{display:block}dl.description>dt,dl.docinfo>dt,dl.field-list>dt{font-weight:400;font-style:italic}dl.description>dt{clear:left;float:left;margin:0;padding:0 .5em 0 0}dl.description>dd:after{display:block;content:"";clear:both}.citation-list,.footnote-list{display:contents}.citation{padding-left:1.5em}.citation .label{margin-left:-1.5em}figure{display:flex;flex-wrap:wrap;align-items:flex-start}figure.fullwidth>img,figure>img{margin:0 .5em .5em 0;padding:0}figcaption{font-size:.8em}.fullwidth>figcaption{font-size:inherit}figure.numbered>figcaption>p:before{counter-increment:figure;content:"Figure " counter(figure) ": "}table.booktabs{border-top:2px solid;border-bottom:2px solid}table.booktabs *{border:0}table.booktabs th{border-bottom:thin solid}table.numbered>caption:before{counter-increment:table;content:"Table " counter(table) ": "}.admonition,.system-message{border-style:solid;border-color:silver;border-width:thin;margin:1em 0;padding:.5em}.attention p.admonition-title,.caution p.admonition-title,.danger p.admonition-title,.warning p.admonition-title,div.error{color:maroon}code .comment,pre.code .comment{color:#5c6576}code .keyword,pre.code .keyword{color:#3b0d06;font-weight:700}code .literal.string,pre.code .literal.string{color:#0c5404}code .name.builtin,pre.code .name.builtin{color:#352b84}code .deleted,pre.code .deleted{background-color:#deb0a1}code .inserted,pre.code .inserted{background-color:#a3d289}.sans{font-family:"Gill Sans","Gill Sans MT",Calibri,"Lucida Sans","Noto Sans",sans-serif;letter-spacing:.02em}a{color:inherit}a:link,a:link:hover{text-decoration:underline}.backrefs a:link,.contents a:link,a.citation-reference:link,a.image-reference:link,a.toc-backref:link,a[href^="#system-message"],a[role=doc-backlink]:link,a[role=doc-noteref]:link{text-decoration:none}.contents>p.topic-title,.fullwidth,footer,h1,h2,h3,header,hr.docutils{clear:both}div.align-left,figure.align-left,img.align-left,svg.align-left,table.align-left,video.align-left{margin-left:0;padding-left:0;padding-right:.5em;clear:left;float:left}figure.align-left>img{margin-left:0;padding-left:0}div.align-right,img.align-right,svg.align-right,video.align-right{padding-left:.5em;clear:right;float:right}figure.align-right{clear:right;float:right}figure.align-right>img{justify-self:right;padding:0}table.align-right{margin-right:2.5%}figure.align-center{align-content:center;justify-content:center}figure.align-center>img{padding-left:0;justify-self:center}.admonition.marginal,.marginal,.topic.marginal,aside.sidebar{background-color:#efefea;box-sizing:border-box;margin-left:2%;margin-right:0;padding:.5em;font-size:.8em}aside.sidebar{background-color:inherit}.footnote{font-size:smaller}@media (min-width:35em){footer,header,main{padding:.5em calc(15% - 3rem);line-height:1.6}.admonition.marginal,.marginal,.topic.marginal,aside.sidebar{max-width:45%;float:right;clear:right}dl.docinfo>dd,dl.field-list>dd,dl.option-list>dd{margin-left:6em}}@media (min-width:65em){main,section{display:grid;grid-template-columns:[content] minmax(0,6fr) [margin] 3fr [end];grid-column-gap:calc(3em + 1%)}main>section,section>section{grid-column:1/end}footer,header,main{padding-right:5%}section>figure{display:contents}.citation.align-left,.footnote.align-left,figure>img,main>*,section>*{grid-column:content}.citation.align-left{font-size:1em;padding-left:1.5em}.citation.align-left .label{margin-left:-1.5em}figure>img{margin:.5em 2%;padding-left:1em}.admonition.marginal,.citation,.footnote,.marginal,.topic.marginal,aside.sidebar,figcaption{grid-column:margin;width:auto;max-width:55em;margin:.5em 0;border:none;padding:0;font-size:.8em;text-align:initial;background-color:inherit}.admonition.marginal{padding:.5em}figure.marginal{display:block;margin:.5em 0}.citation,.footnote{padding-left:0}.citation .label,.footnote .label{margin-left:0}.fullwidth,.fullwidth figcaption,.fullwidth img,aside.system-message,div.abstract,div.dedication,dl.docinfo,h1.title,nav.contents,p.subtitle,pre{grid-column:content/end;margin-right:calc(10% - 3rem);max-width:55em}}@media (min-width:100em){footer,header,main{padding-left:30%}main>nav.contents{position:fixed;top:0;left:0;box-sizing:border-box;width:25%;height:100vh;margin:0;background-color:#fafaf6;padding:5.5em 2%;overflow:auto}main>nav.contents>*{padding-left:0}}WebwareForPython-DBUtils-8de754c/docs/main.de.html000066400000000000000000001334051524214146500220530ustar00rootroot00000000000000 Benutzeranleitung für DBUtils

Benutzeranleitung für DBUtils

Version:
3.2.0
Translations:

English | German

Zusammenfassung

DBUtils ist eine Sammlung von Python-Modulen, mit deren Hilfe man in Python geschriebene Multithread-Anwendungen auf sichere und effiziente Weise an Datenbanken anbinden kann.

DBUtils wurde ursprünglich speziell für Webware for Python als Anwendung und PyGreSQL als PostgreSQL-Datenbankadapter entwickelt, kann aber inzwischen für beliebige Python-Anwendungen und beliebige auf DB-API 2 beruhende Python-Datenbankadapter verwendet werden.

Module

DBUtils ist als Python-Package realisiert worden, das aus zwei verschiedenen Gruppen von Modulen besteht: Einer Gruppe zur Verwendung mit beliebigen DB-API-2-Datenbankadaptern, und einer Gruppe zur Verwendung mit dem klassischen PyGreSQL-Datenbankadapter-Modul.

Allgemeine Variante für beliebige DB-API-2-Adapter

steady_db

Gehärtete DB-API-2-Datenbankverbindungen

pooled_db

Pooling für DB-API-2-Datenbankverbindungen

persistent_db

Persistente DB-API-2-Datenbankverbindungen

simple_pooled_db

Einfaches Pooling für DB-API 2

Variante speziell für den klassischen PyGreSQL-Adapter

steady_pg

Gehärtete klassische PyGreSQL-Verbindungen

pooled_pg

Pooling für klassische PyGreSQL-Verbindungen

persistent_pg

Persistente klassische PyGreSQL-Verbindungen

simple_pooled_pg

Einfaches Pooling für klassisches PyGreSQL

Die Abhängigkeiten der Module in der Variante für beliebige DB-API-2-Adapter sind im folgenden Diagramm dargestellt:

dependencies_db.png

Die Abhängigkeiten der Module in der Variante für den klassischen PyGreSQL-Adapter sehen ähnlich aus:

dependencies_pg.png

Download

Die aktuelle Version von DBUtils kann vom Python Package Index heruntergeladen werden:

https://pypi.python.org/pypi/DBUtils

Das Source-Code-Repository befindet sich hier auf GitHub:

https://github.com/WebwareForPython/DBUtils

Installation

Das Paket kann auf die übliche Weise installiert werden:

python setup.py install

Noch einfacher ist, das Paket in einem Schritt mit pip automatisch herunterzuladen und zu installieren:

pip install DBUtils

Anforderungen

DBUtils unterstützt die Python Versionen 3.7 bis 3.14.

Die Module in der Variante für klassisches PyGreSQL benötigen PyGreSQL Version 4.0 oder höher, während die Module in der allgemeinen Variante für DB-API 2 mit jedem beliebigen Python-Datenbankadapter-Modul zusammenarbeiten, das auf DB-API 2 basiert.

Funktionalität

Dieser Abschnitt verwendet nur die Bezeichnungen der DB-API-2-Variante, aber Entsprechendes gilt auch für die PyGreSQL-Variante.

DBUtils installiert sich als Paket dbutils, das alle hier beschriebenen Module enthält. Jedes dieser Module enthält im Wesentlichen eine Klasse, die einen analogen Namen trägt und die jeweilige Funktionalität bereitstellt. So enthält z.B. das Modul dbutils.pooled_db die Klasse PooledDB.

SimplePooledDB (simple_pooled_db)

Die Klasse SimplePooledDB in dbutils.simple_pooled_db ist eine sehr elementare Referenz-Implementierung eines Pools von Datenbankverbindungen. Hiermit ist ein Vorratsspeicher an Datenbankverbindungen gemeint, aus dem sich die Python-Anwendung bedienen kann. Diese Implementierung ist weit weniger ausgefeilt als das eigentliche pooled_db-Modul und stellt insbesondere keine Ausfallsicherung zur Verfügung. dbutils.simple_pooled_db ist im Wesentlichen identisch mit dem zu Webware for Python gehörenden Modul MiscUtils.DBPool. Es ist eher zur Verdeutlichung des Konzepts gedacht, als zum Einsatz im produktiven Betrieb.

SteadyDBConnection (steady_db)

Die Klasse SteadyDBConnection im Modul dbutils.steady_db stellt "gehärtete" Datenbankverbindungen bereit, denen gewöhnliche Verbindungen eines DB-API-2-Datenbankadapters zugrunde liegen. Eine "gehärtete" Verbindung wird bei Zugriff automatisch, ohne dass die Anwendung dies bemerkt, wieder geöffnet, wenn sie geschlossen wurde, die Datenbankverbindung unterbrochen wurde, oder wenn sie öfter als ein optionales Limit genutzt wurde.

Ein typisches Beispiel, wo dies benötigt wird, ist, wenn die Datenbank neu gestartet wurde, während Ihre Anwendung immer noch läuft und Verbindungen zur Datenbank offen hat, oder wenn Ihre Anwendung auf eine entfernte Datenbank über ein Netzwerk zugreift, das durch eine Firewall geschützt ist, und die Firewall neu gestartet wurde und dabei ihren Verbindungsstatus verloren hat.

Normalerweise benutzen Sie das steady_db-Modul nicht direkt; es wird aber von den beiden nächsten Modulen benötigt, persistent_db und pooled_db.

PersistentDB (persistent_db)

Die Klasse PersistentDB im Modul dbutils.persistent_db stellt gehärtete, thread-affine, persistente Datenbankverbindungen zur Verfügung, unter Benutzung eines beliebigen DB-API-2-Datenbankadapters. Mit "thread-affin" und "persistent" ist hierbei gemeint, dass die einzelnen Datenbankverbindungen den jeweiligen Threads fest zugeordnet bleiben und während der Laufzeit des Threads nicht geschlossen werden.

Das folgende Diagramm zeigt die beteiligten Verbindungsschichten, wenn Sie persistent_db-Datenbankverbindungen einsetzen:

persistent.png

Immer wenn ein Thread eine Datenbankverbindung zum ersten Mal öffnet, wird eine neue Datenbankverbindung geöffnet, die von da an immer wieder für genau diesen Thread verwendet wird. Wenn der Thread die Datenbankverbindung schließt, wird sie trotzdem weiter offen gehalten, damit beim nächsten Mal, wenn der gleiche Thread wieder eine Datenbankverbindung anfordert, diese gleiche bereits geöffnete Datenbankverbindung wieder verwendet werden kann. Die Verbindung wird automatisch geschlossen, wenn der Thread beendet wird.

Kurz gesagt versucht persistent_db Datenbankverbindungen wiederzuverwerten, um die Gesamteffizienz der Datenbankzugriffe Ihrer Multithread-Anwendungen zu steigern, aber es wird dabei sichergestellt, dass verschiedene Threads niemals die gleiche Verbindung benutzen.

Daher arbeitet persistent_db sogar dann problemlos, wenn der zugrunde liegende DB-API-2-Datenbankadapter nicht thread-sicher auf der Verbindungsebene ist, oder wenn parallele Threads Parameter der Datenbank-Sitzung verändern oder Transaktionen mit mehreren SQL-Befehlen durchführen.

PooledDB (pooled_db)

Die Klasse PooledDB im Modul dbutils.pooled_db stellt, unter Benutzung eines beliebigen DB-API-2-Datenbankadapters, einen Pool von gehärteten, thread-sicheren Datenbankverbindungen zur Verfügung, die automatisch, ohne dass die Anwendung dies bemerkt, wiederverwendet werden.

Das folgende Diagramm zeigt die beteiligten Verbindungsschichten, wenn Sie pooled_db-Datenbankverbindungen einsetzen:

pooled.png

Wie im Diagramm angedeutet, kann pooled_db geöffnete Datenbankverbindungen den verschiedenen Threads beliebig zuteilen. Dies geschieht standardmäßig, wenn Sie den Verbindungspool mit einem positiven Wert für maxshared einrichten und der zugrunde liegende DB-API-2-Datenbankadapter auf der Verbindungsebene thread-sicher ist, aber Sie können auch dedizierte Datenbankverbindungen anfordern, die nicht von anderen Threads verwendet werden sollen. Neben dem Pool gemeinsam genutzter Datenbankverbindungen ("shared pool") können Sie auch einen Pool von mindestens mincached und höchstens maxcached inaktiven Verbindungen auf Vorrat einrichten ("idle pool"), aus dem immer dann geschöpft wird, wenn ein Thread eine dedizierte Datenbankverbindung anfordert, oder wenn der Pool gemeinsam genutzter Datenbankverbindungen noch nicht voll ist. Wenn ein Thread eine Datenbankverbindung schließt, die auch von keinem anderen Thread mehr benutzt wird, wird sie an den Vorratsspeicher inaktiver Datenbankverbindungen zurückgegeben, damit sie wiederverwertet werden kann.

Wenn der zugrunde liegende DB-API-Datenbankadapter nicht thread-sicher ist, werden Thread-Locks verwendet, um sicherzustellen, dass die pooled_db-Verbindungen dennoch thread-sicher sind. Sie brauchen sich also hierum keine Sorgen zu machen, aber Sie sollten darauf achten, dedizierte Datenbankverbindungen zu verwenden, sobald Sie Parameter der Datenbanksitzung verändern oder Transaktionen mit mehreren SQL-Befehlen ausführen.

Die Qual der Wahl

Sowohl persistent_db als auch pooled_db dienen dem gleichen Zweck, nämlich die Effizienz des Datenbankzugriffs durch Wiederverwendung von Datenbankverbindungen zu steigern, und dabei gleichzeitig die Stabilität zu gewährleisten, selbst wenn die Datenbankverbindung unterbrochen wird.

Welches der beiden Module sollte also verwendet werden? Nach den obigen Erklärungen ist es klar, dass persistent_db dann sinnvoller ist, wenn Ihre Anwendung eine gleich bleibende Anzahl Threads verwendet, die häufig auf die Datenbank zugreifen. In diesem Fall werden Sie ungefähr die gleiche Anzahl geöffneter Datenbankverbindungen erhalten. Wenn jedoch Ihre Anwendung häufig Threads beendet und neu startet, dann ist pooled_db die bessere Lösung, die auch mehr Möglichkeiten zur Feineinstellung zur Verbesserung der Effizienz erlaubt, insbesondere bei Verwendung eines thread-sicheren DB-API-2-Datenbankadapters.

Da die Schnittstellen beider Module sehr ähnlich sind, können Sie recht einfach von einem Modul zum anderen wechseln und austesten, welches geeigneter ist.

Benutzung

Die Benutzung aller Module ist zwar recht ähnlich, aber es gibt vor allem bei der Initialisierung auch einige Unterschiede, sowohl zwischen den "Pooled"- und den "Persistent"-Varianten, als auch zwischen den DB-API-2- und den PyGreSQL-Varianten.

Wir werden hier nur auf das persistent_db-Modul und das etwas kompliziertere pooled_db-Modul eingehen. Einzelheiten zu den anderen Modulen finden Sie in deren Docstrings. Unter Verwendung der Python-Interpreter-Konsole können Sie sich die Dokumentation des pooled_db-Moduls wie folgt anzeigen lassen (dies funktioniert entsprechend auch mit den anderen Modulen):

help(pooled_db)

PersistentDB (persistent_db)

Wenn Sie das persistent_db-Modul einsetzen möchten, müssen Sie zuerst einen Generator für die von Ihnen gewünschte Art von Datenbankverbindungen einrichten, indem Sie eine Instanz der Klasse PersistentDB erzeugen, wobei Sie folgende Parameter angeben müssen:

  • creator: entweder eine Funktion, die neue DB-API-2-Verbindungen erzeugt, oder ein DB-API-2-Datenbankadapter-Modul

  • maxusage: Obergrenze dafür, wie oft eine einzelne Verbindung wiederverwendet werden darf (der Standardwert 0 oder None bedeutet unbegrenzte Wiederverwendung)

    Sobald diese Obergrenze erreicht wird, wird die Verbindung zurückgesetzt.

  • setsession: eine optionale Liste von SQL-Befehlen zur Initialisierung der Datenbanksitzung, z.B. ["set datestyle to german", ...]

  • failures: eine optionale Exception-Klasse oder ein Tupel von Exceptions, bei denen die Ausfallsicherung zum Tragen kommen soll, falls die Vorgabe (OperationalError, InterfaceError, InternalError) für das verwendete Datenbankadapter-Modul nicht geeignet sein sollte

  • isfatal: eine optionale Funktion, der die aufgetretene Exception übergeben wird und die zurückgibt, ob die Ausfallsicherung zum Tragen kommen soll (muss als Schlüsselwort-Argument übergeben werden, siehe Steuerung der Ausfallsicherung)

  • ping: mit diesem Parameter kann eingestellt werden, wann Verbindungen geprüft werden (0 = None = nie, 1 = Standardwert = immer wenn neu angefragt, 2 = vor Erzeugen eines Cursors, 4 = vor dem Ausführen von Abfragen, 7 = immer, und alle Bitkombinationen dieser Werte)

    Standardmäßig werden Verbindungen mit der ping()-Methode geprüft, falls eine solche vorhanden ist. Sie können für die Prüfung stattdessen auch einen SQL-Befehl oder eine Funktion angeben, die dann wie bei ping=1 erfolgt, oder ein Tupel aus dem obigen Wert und einem der beiden (siehe Prüfung der Verbindungen).

  • closeable: wenn dies auf True gesetzt wird, dann wird das Schließen von Verbindungen erlaubt, normalerweise wird es jedoch ignoriert

  • threadlocal: eine optionale Klasse zur Speicherung thread-lokaler Daten, die anstelle unserer Python-Implementierung benutzt wird (threading.local ist schneller, kann aber nicht in allen Fällen verwendet werden)

  • Die als creator angegebene Funktion oder die Funktion connect des DB-API-2-Datenbankadapter-Moduls erhalten alle weiteren Parameter, wie host, database, user, password usw. Sie können einige oder alle dieser Parameter in Ihrer eigenen creator-Funktion setzen, was ausgefeilte Mechanismen zur Ausfallsicherung und Lastverteilung ermöglicht.

Wenn Sie beispielsweise pgdb als DB-API-2-Datenbankadapter verwenden, und möchten, dass jede Verbindung Ihrer lokalen Datenbank meinedb 1000 mal wiederverwendet werden soll, sieht die Initialisierung so aus:

import pgdb  # importiere das verwendete DB-API-2-Modul
from dbutils.persistent_db import PersistentDB
persist = PersistentDB(pgdb, 1000, database='meinedb')

Nachdem Sie den Generator mit diesen Parametern eingerichtet haben, können Sie derartige Datenbankverbindungen von da an wie folgt anfordern:

db = persist.connection()

Sie können diese Verbindungen verwenden, als wären sie gewöhnliche DB-API-2-Datenbankverbindungen. Genauer genommen erhalten Sie die "gehärtete" steady_db-Version der zugrunde liegenden DB-API-2-Verbindung.

Wenn Sie eine solche persistente Verbindung mit db.close() schließen, wird dies stillschweigend ignoriert, denn sie würde beim nächsten Zugriff sowieso wieder geöffnet, und das wäre nicht im Sinne persistenter Verbindungen. Stattdessen wird die Verbindung automatisch dann geschlossen, wenn der Thread endet. Sie können dieses Verhalten ändern, indem Sie den Parameter namens closeable setzen.

Das Holen einer Verbindung kann etwas beschleunigt werden, indem man den Parameter threadlocal auf threading.local setzt; dies könnte aber in einigen Umgebungen nicht funktionieren (es ist zum Beispiel bekannt, dass mod_wsgi hier Probleme bereitet, da es Daten, die mit threading.local gespeichert wurden, zwischen Requests löscht).

PooledDB (pooled_db)

Wenn Sie das pooled_db-Modul einsetzen möchten, müssen Sie zuerst einen Pool für die von Ihnen gewünschte Art von Datenbankverbindungen einrichten, indem Sie eine Instanz der Klasse PooledDB erzeugen, wobei Sie folgende Parameter angeben müssen:

  • creator: entweder eine Funktion, die neue DB-API-2-Verbindungen erzeugt, oder ein DB-API-2-Datenbankadapter-Modul

  • mincached: die anfängliche Anzahl inaktiver Verbindungen, die auf Vorrat gehalten werden sollen (der Standardwert 0 bedeutet, dass beim Start keine Verbindungen geöffnet werden)

  • maxcached: Obergrenze für die Anzahl inaktiver Verbindungen, die auf Vorrat gehalten werden sollen (der Standardwert 0 oder None bedeutet unbegrenzte Größe des Vorratsspeichers)

  • maxshared: Obergrenze für die Anzahl gemeinsam genutzter Verbindungen (der Standardwert 0 oder None bedeutet, dass alle Verbindungen dediziert sind)

    Wenn diese Obergrenze erreicht wird, werden Verbindungen gemeinsam genutzt, sofern diese als gemeinsam nutzbar angefordert werden.

  • maxconnections: Obergrenze für die Anzahl an Datenbankverbindungen, die insgesamt überhaupt erlaubt werden sollen (der Standardwert 0 oder None bedeutet unbegrenzte Anzahl von Datenbankverbindungen)

  • blocking: bestimmt das Verhalten bei Ãœberschreitung dieser Obergrenze

    Wenn dies auf True gesetzt wird, dann wird so lange gewartet, bis die Anzahl an Datenbankverbindungen wieder abnimmt, normalerweise wird jedoch sofort eine Fehlermeldung ausgegeben.

  • maxusage: Obergrenze dafür, wie oft eine einzelne Verbindung wiederverwendet werden darf (der Standardwert 0 oder None bedeutet unbegrenzte Wiederverwendung)

    Sobald diese Obergrenze erreicht wird, wird die Verbindung automatisch zurückgesetzt (geschlossen und wieder neu geöffnet).

  • setsession: eine optionale Liste von SQL-Befehlen zur Initialisierung der Datenbanksitzung, z.B. ["set datestyle to german", ...]

  • reset: wie Verbindungen zurückgesetzt werden sollen, bevor sie wieder in den Verbindungspool zurückgegeben werden (False oder None um mit begin() gestartete Transaktionen zurückzurollen, der Standardwert True rollt sicherheitshalber mögliche Transaktionen immer zurück)

  • failures: eine optionale Exception-Klasse oder ein Tupel von Exceptions, bei denen die Ausfallsicherung zum Tragen kommen soll, falls die Vorgabe (OperationalError, InterfaceError, InternalError) für das verwendete Datenbankadapter-Modul nicht geeignet sein sollte

  • isfatal: eine optionale Funktion, der die aufgetretene Exception übergeben wird und die zurückgibt, ob die Ausfallsicherung zum Tragen kommen soll (muss als Schlüsselwort-Argument übergeben werden, siehe Steuerung der Ausfallsicherung)

  • ping: mit diesem Parameter kann eingestellt werden, wann Verbindungen geprüft werden (0 = None = nie, 1 = Standardwert = immer wenn neu angefragt, 2 = vor Erzeugen eines Cursors, 4 = vor dem Ausführen von Abfragen, 7 = immer, und alle Bitkombinationen dieser Werte)

    Standardmäßig werden Verbindungen mit der ping()-Methode geprüft, falls eine solche vorhanden ist. Sie können für die Prüfung stattdessen auch einen SQL-Befehl oder eine Funktion angeben, die dann wie bei ping=1 erfolgt, oder ein Tupel aus dem obigen Wert und einem der beiden (siehe Prüfung der Verbindungen).

  • Die als creator angegebene Funktion oder die Funktion connect des DB-API-2-Datenbankadapter-Moduls erhalten alle weiteren Parameter, wie host, database, user, password usw. Sie können einige oder alle dieser Parameter in Ihrer eigenen creator-Funktion setzen, was ausgefeilte Mechanismen zur Ausfallsicherung und Lastverteilung ermöglicht.

Wenn Sie beispielsweise pgdb als DB-API-2-Datenbankadapter benutzen, und einen Pool von mindestens fünf Datenbankverbindungen zu Ihrer Datenbank meinedb verwenden möchten, dann sieht die Initialisierung so aus:

import pgdb  # importiere das verwendete DB-API-2-Modul
from dbutils.pooled_db import PooledDB
pool = PooledDB(pgdb, 5, database='meinedb')

Nachdem Sie den Pool für Datenbankverbindungen so eingerichtet haben, können Sie Verbindungen daraus wie folgt anfordern:

db = pool.connection()

Sie können diese Verbindungen verwenden, als wären sie gewöhnliche DB-API-2-Datenbankverbindungen. Genauer genommen erhalten Sie die "gehärtete" steady_db-Version der zugrunde liegenden DB-API-2-Verbindung.

Bitte beachten Sie, dass die Verbindung von anderen Threads mitgenutzt werden kann, wenn Sie den Parameter maxshared auf einen Wert größer als Null gesetzt haben, und der zugrunde liegende DB-API-2-Datenbankadapter dies erlaubt. Eine dedizierte Datenbankverbindung, die garantiert nicht von anderen Threads mitgenutzt wird, fordern Sie wie folgt an:

db = pool.connection(shareable=False)

Stattdessen können Sie eine dedizierte Verbindung auch wie folgt erhalten:

db = pool.dedicated_connection()

Wenn Sie die Datenbankverbindung nicht mehr benötigen, sollten Sie diese sofort wieder mit db.close() an den Pool zurückgeben. Sie können auf die gleiche Weise eine neue Verbindung erhalten.

Sie können dies auch durch Verwendung von Kontext-Managern vereinfachen:

with pool.connection() as db:
    with db.cursor() as cur:
        cur.execute(...)
        res = cur.fetchone()

Besonderheiten bei der Benutzung

Manchmal möchte man Datenbankverbindungen besonders vorbereiten, bevor sie von DBUtils verwendet werden, und dies ist nicht immer durch Verwendung der passenden Parameter möglich. Zum Beispiel kann es pyodbc erfordern, dass man die Methode setencoding() der Datenbankverbindung aufruft. Sie können dies erreichen, indem Sie eine modifizierte Version der Funktion connect() verwenden und diese als creator (dem ersten Argument) an PersistentDB oder PooledDB übergeben, etwa so:

import pyodbc
from dbutils.pooled_db import PooledDB

def creator():
    con = pyodbc.connect(...)
    con.setencoding(...)
    return con

creator.dbapi = pyodbc

db_pool = PooledDB(creator, mincached=5)

Steuerung der Ausfallsicherung

Wenn eine Datenbankoperation mit einer der im Parameter failures angegebenen Exception-Klassen fehlschlägt, dann nimmt DBUtils an, dass die Verbindung verloren gegangen ist, und wiederholt die Operation transparent, zunächst mit einem neuen Cursor und danach mit einer neu geöffneten Verbindung. Genau das möchte man, wenn der Datenbankserver neu gestartet wurde, aber nicht, wenn die Anweisung selbst aus einem Grund fehlgeschlagen ist, der nichts mit dem Zustand der Verbindung zu tun hat.

Leider melden viele Datenbankadapter-Module beide Arten von Problemen mit derselben Exception-Klasse. Wenn zum Beispiel MySQL eine Anweisung abbricht, weil diese eine absichtlich angeforderte Zeitbegrenzung überschritten hat:

cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...')

dann löst pymysql einen OperationalError aus, genau wie beim tatsächlichen Verlust der Verbindung. Da der Parameter failures nur Exception-Klassen angeben kann und Python Unterklassen nicht von einer except-Klausel ausschließen kann, lässt sich diese Unterscheidung nicht durch eine Einschränkung des Parameters failures ausdrücken. Im obigen Beispiel würde die Anweisung bis zu dreimal ausgeführt, und die Abfrage würde bis zu 30 statt 10 Sekunden dauern.

Für solche Fälle können Sie eine Funktion als isfatal übergeben, die für jeden zu failures passenden Fehler gefragt wird, ob dieser für die Verbindung wirklich fatal ist:

import pymysql
from dbutils.pooled_db import PooledDB

def no_timeout(error):
    # 3024 = Zeitbegrenzung überschritten, die Verbindung ist noch intakt
    return not error.args or error.args[0] != 3024

pool = PooledDB(pymysql, 5, database='mydb', isfatal=no_timeout)

Wenn die Funktion einen falschen Wert zurückgibt, dann gilt der Fehler als nicht fatal, wird also sofort an die Anwendung weitergegeben, und die Verbindung wird weder ersetzt noch aus dem Verbindungspool entfernt. Beachten Sie, dass isfatal als Schlüsselwort-Argument übergeben werden muss und mit beliebigen Exceptions der als failures angegebenen Klassen umgehen können muss.

Wenn Sie die Ausfallsicherung nur für eine bestimmte Anweisung abschalten möchten, statt sie für den gesamten Verbindungspool zu konfigurieren, dann können Sie den Kontextmanager no_failover() des Cursors verwenden:

with pool.connection() as db:
    with db.cursor() as cur:
        with cur.no_failover():
            cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...')
        rows = cur.fetchall()

Alles Übrige, insbesondere die ping()-Prüfung und die Zählung der Verwendungen der zugrunde liegenden Verbindung, bleibt innerhalb dieses Kontexts unverändert.

Als letztes Mittel können Sie über das Attribut dbapi_connection einer Verbindung und das Attribut dbapi_cursor eines Cursors auch direkt auf die zugrunde liegenden DB-API-2-Objekte zugreifen. Beachten Sie, dass Operationen auf diesen Objekten überhaupt nicht von der Ausfallsicherung erfasst und auch nicht auf die Begrenzung maxusage angerechnet werden.

Prüfung der Verbindungen

Verbindungen, die im Verbindungspool gehalten oder einem Thread zugeordnet sind, können unbrauchbar werden, wenn der Datenbankserver neu gestartet wird oder zu lange untätige Verbindungen abbricht. Damit dies auffällt, bevor die Verbindung verwendet wird, kann DBUtils sie mit der ping()-Methode der zugrunde liegenden DB-API-2-Verbindung prüfen, wobei der Parameter ping festlegt, wann eine solche Prüfung stattfindet.

Allerdings ist ping() nicht Teil der DB-API-2-Spezifikation, und viele gebräuchliche Datenbankadapter-Module stellen diese Methode nicht bereit. Sobald DBUtils bemerkt, dass die Methode nicht verfügbar ist, werden die Verbindungen überhaupt nicht mehr geprüft. In diesem Fall können Sie als Parameter ping einen SQL-Befehl angeben, der stattdessen zur Prüfung der Verbindung ausgeführt werden soll:

import psycopg2
from dbutils.pooled_db import PooledDB

pool = PooledDB(psycopg2, 5, database='meinedb', ping='select 1')

Statt eines SQL-Befehls können Sie auch eine Funktion angeben, der die zugrunde liegende DB-API-2-Verbindung übergeben wird und die zurückgibt, ob diese noch intakt ist:

def is_healthy(con):
    # psycopg2 weiß, ob die Verbindung bereits geschlossen wurde
    return not con.closed

pool = PooledDB(psycopg2, 5, database='meinedb', ping=is_healthy)

Die Rückgabe von None gilt als intakt, das Auslösen einer Exception als nicht intakt, so dass auch eine Funktion verwendet werden kann, die einfach eine Datenbankoperation ausführt, ohne Fehler abzufangen. Meldet die Prüfung die Verbindung als nicht intakt, dann wird diese transparent durch eine neu geöffnete Verbindung ersetzt, genau wie wenn ping() sie als nicht mehr brauchbar gemeldet hätte.

Eine solche Prüfung findet immer dann statt, wenn eine Verbindung aus dem Verbindungspool geholt oder von einem Thread angefragt wird, also zu dem Zeitpunkt, der dem Wert ping=1 entspricht. Wenn sie auch zu den anderen Zeitpunkten stattfinden soll, dann geben Sie ein Tupel aus dem üblichen ganzzahligen Wert und dem SQL-Befehl oder der Funktion an:

pool = PooledDB(psycopg2, 5, database='meinedb', ping=(4, 'select 1'))

Anmerkungen

Wenn Sie einen der bekannten "Object-Relational Mapper" SQLObject oder SQLAlchemy verwenden, dann benötigen Sie DBUtils nicht, denn diese haben ihre eigenen Mechanismen zum Pooling von Datenbankverbindungen eingebaut. Tatsächlich hat SQLObject 2 (SQL-API) das Pooling in eine separate Schicht ausgelagert, in der Code von DBUtils verwendet wird.

Wenn Sie eine Lösung verwenden wie den Apache-Webserver mit mod_python oder mod_wsgi, dann sollten Sie bedenken, dass Ihr Python-Code normalerweise im Kontext der Kindprozesse des Webservers läuft. Wenn Sie also das pooled_db-Modul einsetzen, und mehrere dieser Kindprozesse laufen, dann werden Sie ebenso viele Pools mit Datenbankverbindungen erhalten. Wenn diese Prozesse viele Threads laufen lassen, dann mag dies ein sinnvoller Ansatz sein, wenn aber diese Prozesse nicht mehr als einen Worker-Thread starten, wie im Fall des Multi-Processing-Moduls "prefork" für den Apache-Webserver, dann sollten Sie auf eine Middleware für das Connection-Pooling zurückgreifen, die Multi-Processing unterstützt, wie zum Beispiel pgpool oder pgbouncer für die PostgreSQL-Datenbank.

Zukunft

Einige Ideen für zukünftige Verbesserungen:

  • Alternativ zur Obergrenze in der Anzahl der Nutzung einer Datenbankverbindung könnte eine maximale Lebensdauer für die Verbindung implementiert werden.

  • Es könnten Module monitor_db und monitor_pg hinzugefügt werden, die in einem separaten Thread ständig den "idle pool" und eventuell auch den "shared pool" bzw. die persistenten Verbindungen überwachen. Wenn eine unterbrochene Datenbankverbindung entdeckt wird, wird diese automatisch durch den Monitor-Thread wiederhergestellt. Dies ist in einem Szenario sinnvoll, bei dem die Datenbank einer Website jede Nacht neu gestartet wird. Ohne den Monitor-Thread würden die Benutzer morgens eine kleine Verzögerung bemerken, weil erst dann die unterbrochenen Datenbankverbindungen entdeckt würden und sich der Pool langsam wieder neu aufbaut. Mit dem Monitor-Thread würde dies schon während der Nacht passieren, kurz nach der Unterbrechung. Der Monitor-Thread könnte auch so konfiguriert werden, dass er generell täglich den Verbindungspool erneuert, kurz bevor die Benutzer erscheinen.

  • Optional sollten Benutzung, schlechte Verbindungen und Ãœberschreitung von Obergrenzen in Logs gespeichert werden können.

Fehlermeldungen und Feedback

Fehlermeldungen, Patches und Feedback können Sie als Issues oder Pull Requests auf der GitHub-Projektseite von DBUtils übermitteln.

Autoren

Autor:

Christoph Zwerschke

Beiträge:

DBUtils benutzt Code, Anmerkungen und Vorschläge von Ian Bicking, Chuck Esterbrook (Webware for Python), Dan Green (DBTools), Jay Love, Michael Palmer, Tom Schwaller, Geoffrey Talvola, Warren Smith (DbConnectionPool), Ezio Vernacotola, Jehiah Czebotar, Matthew Harriger, Gregory Piñero und Josef van Eenbergen.

WebwareForPython-DBUtils-8de754c/docs/main.de.rst000066400000000000000000001027021524214146500217130ustar00rootroot00000000000000Benutzeranleitung für DBUtils +++++++++++++++++++++++++++++ :Version: 3.2.0 :Translations: English_ | German .. _English: main.html .. contents:: Inhalt Zusammenfassung =============== DBUtils_ ist eine Sammlung von Python-Modulen, mit deren Hilfe man in Python_ geschriebene Multithread-Anwendungen auf sichere und effiziente Weise an Datenbanken anbinden kann. DBUtils wurde ursprünglich speziell für `Webware for Python`_ als Anwendung und PyGreSQL_ als PostgreSQL_-Datenbankadapter entwickelt, kann aber inzwischen für beliebige Python-Anwendungen und beliebige auf `DB-API 2`_ beruhende Python-Datenbankadapter verwendet werden. Module ====== DBUtils ist als Python-Package realisiert worden, das aus zwei verschiedenen Gruppen von Modulen besteht: Einer Gruppe zur Verwendung mit beliebigen DB-API-2-Datenbankadaptern, und einer Gruppe zur Verwendung mit dem klassischen PyGreSQL-Datenbankadapter-Modul. +------------------+----------------------------------------------+ | Allgemeine Variante für beliebige DB-API-2-Adapter | +==================+==============================================+ | steady_db | Gehärtete DB-API-2-Datenbankverbindungen | +------------------+----------------------------------------------+ | pooled_db | Pooling für DB-API-2-Datenbankverbindungen | +------------------+----------------------------------------------+ | persistent_db | Persistente DB-API-2-Datenbankverbindungen | +------------------+----------------------------------------------+ | simple_pooled_db | Einfaches Pooling für DB-API 2 | +------------------+----------------------------------------------+ +------------------+----------------------------------------------+ | Variante speziell für den klassischen PyGreSQL-Adapter | +==================+==============================================+ | steady_pg | Gehärtete klassische PyGreSQL-Verbindungen | +------------------+----------------------------------------------+ | pooled_pg | Pooling für klassische PyGreSQL-Verbindungen | +------------------+----------------------------------------------+ | persistent_pg | Persistente klassische PyGreSQL-Verbindungen | +------------------+----------------------------------------------+ | simple_pooled_pg | Einfaches Pooling für klassisches PyGreSQL | +------------------+----------------------------------------------+ Die Abhängigkeiten der Module in der Variante für beliebige DB-API-2-Adapter sind im folgenden Diagramm dargestellt: .. image:: dependencies_db.png Die Abhängigkeiten der Module in der Variante für den klassischen PyGreSQL-Adapter sehen ähnlich aus: .. image:: dependencies_pg.png Download ======== Die aktuelle Version von DBUtils kann vom Python Package Index heruntergeladen werden:: https://pypi.python.org/pypi/DBUtils Das Source-Code-Repository befindet sich hier auf GitHub:: https://github.com/WebwareForPython/DBUtils Installation ============ Das Paket kann auf die übliche Weise installiert werden:: python setup.py install Noch einfacher ist, das Paket in einem Schritt mit `pip`_ automatisch herunterzuladen und zu installieren:: pip install DBUtils .. _pip: https://pip.pypa.io/ Anforderungen ============= DBUtils unterstützt die Python_ Versionen 3.7 bis 3.14. Die Module in der Variante für klassisches PyGreSQL benötigen PyGreSQL_ Version 4.0 oder höher, während die Module in der allgemeinen Variante für DB-API 2 mit jedem beliebigen Python-Datenbankadapter-Modul zusammenarbeiten, das auf `DB-API 2`_ basiert. Funktionalität ============== Dieser Abschnitt verwendet nur die Bezeichnungen der DB-API-2-Variante, aber Entsprechendes gilt auch für die PyGreSQL-Variante. DBUtils installiert sich als Paket ``dbutils``, das alle hier beschriebenen Module enthält. Jedes dieser Module enthält im Wesentlichen eine Klasse, die einen analogen Namen trägt und die jeweilige Funktionalität bereitstellt. So enthält z.B. das Modul ``dbutils.pooled_db`` die Klasse ``PooledDB``. SimplePooledDB (simple_pooled_db) --------------------------------- Die Klasse ``SimplePooledDB`` in ``dbutils.simple_pooled_db`` ist eine sehr elementare Referenz-Implementierung eines Pools von Datenbankverbindungen. Hiermit ist ein Vorratsspeicher an Datenbankverbindungen gemeint, aus dem sich die Python-Anwendung bedienen kann. Diese Implementierung ist weit weniger ausgefeilt als das eigentliche ``pooled_db``-Modul und stellt insbesondere keine Ausfallsicherung zur Verfügung. ``dbutils.simple_pooled_db`` ist im Wesentlichen identisch mit dem zu Webware for Python gehörenden Modul ``MiscUtils.DBPool``. Es ist eher zur Verdeutlichung des Konzepts gedacht, als zum Einsatz im produktiven Betrieb. SteadyDBConnection (steady_db) ------------------------------ Die Klasse ``SteadyDBConnection`` im Modul ``dbutils.steady_db`` stellt "gehärtete" Datenbankverbindungen bereit, denen gewöhnliche Verbindungen eines DB-API-2-Datenbankadapters zugrunde liegen. Eine "gehärtete" Verbindung wird bei Zugriff automatisch, ohne dass die Anwendung dies bemerkt, wieder geöffnet, wenn sie geschlossen wurde, die Datenbankverbindung unterbrochen wurde, oder wenn sie öfter als ein optionales Limit genutzt wurde. Ein typisches Beispiel, wo dies benötigt wird, ist, wenn die Datenbank neu gestartet wurde, während Ihre Anwendung immer noch läuft und Verbindungen zur Datenbank offen hat, oder wenn Ihre Anwendung auf eine entfernte Datenbank über ein Netzwerk zugreift, das durch eine Firewall geschützt ist, und die Firewall neu gestartet wurde und dabei ihren Verbindungsstatus verloren hat. Normalerweise benutzen Sie das ``steady_db``-Modul nicht direkt; es wird aber von den beiden nächsten Modulen benötigt, ``persistent_db`` und ``pooled_db``. PersistentDB (persistent_db) ---------------------------- Die Klasse ``PersistentDB`` im Modul ``dbutils.persistent_db`` stellt gehärtete, thread-affine, persistente Datenbankverbindungen zur Verfügung, unter Benutzung eines beliebigen DB-API-2-Datenbankadapters. Mit "thread-affin" und "persistent" ist hierbei gemeint, dass die einzelnen Datenbankverbindungen den jeweiligen Threads fest zugeordnet bleiben und während der Laufzeit des Threads nicht geschlossen werden. Das folgende Diagramm zeigt die beteiligten Verbindungsschichten, wenn Sie ``persistent_db``-Datenbankverbindungen einsetzen: .. image:: persistent.png Immer wenn ein Thread eine Datenbankverbindung zum ersten Mal öffnet, wird eine neue Datenbankverbindung geöffnet, die von da an immer wieder für genau diesen Thread verwendet wird. Wenn der Thread die Datenbankverbindung schließt, wird sie trotzdem weiter offen gehalten, damit beim nächsten Mal, wenn der gleiche Thread wieder eine Datenbankverbindung anfordert, diese gleiche bereits geöffnete Datenbankverbindung wieder verwendet werden kann. Die Verbindung wird automatisch geschlossen, wenn der Thread beendet wird. Kurz gesagt versucht ``persistent_db`` Datenbankverbindungen wiederzuverwerten, um die Gesamteffizienz der Datenbankzugriffe Ihrer Multithread-Anwendungen zu steigern, aber es wird dabei sichergestellt, dass verschiedene Threads niemals die gleiche Verbindung benutzen. Daher arbeitet ``persistent_db`` sogar dann problemlos, wenn der zugrunde liegende DB-API-2-Datenbankadapter nicht thread-sicher auf der Verbindungsebene ist, oder wenn parallele Threads Parameter der Datenbank-Sitzung verändern oder Transaktionen mit mehreren SQL-Befehlen durchführen. PooledDB (pooled_db) -------------------- Die Klasse ``PooledDB`` im Modul ``dbutils.pooled_db`` stellt, unter Benutzung eines beliebigen DB-API-2-Datenbankadapters, einen Pool von gehärteten, thread-sicheren Datenbankverbindungen zur Verfügung, die automatisch, ohne dass die Anwendung dies bemerkt, wiederverwendet werden. Das folgende Diagramm zeigt die beteiligten Verbindungsschichten, wenn Sie ``pooled_db``-Datenbankverbindungen einsetzen: .. image:: pooled.png Wie im Diagramm angedeutet, kann ``pooled_db`` geöffnete Datenbankverbindungen den verschiedenen Threads beliebig zuteilen. Dies geschieht standardmäßig, wenn Sie den Verbindungspool mit einem positiven Wert für ``maxshared`` einrichten und der zugrunde liegende DB-API-2-Datenbankadapter auf der Verbindungsebene thread-sicher ist, aber Sie können auch dedizierte Datenbankverbindungen anfordern, die nicht von anderen Threads verwendet werden sollen. Neben dem Pool gemeinsam genutzter Datenbankverbindungen ("shared pool") können Sie auch einen Pool von mindestens ``mincached`` und höchstens ``maxcached`` inaktiven Verbindungen auf Vorrat einrichten ("idle pool"), aus dem immer dann geschöpft wird, wenn ein Thread eine dedizierte Datenbankverbindung anfordert, oder wenn der Pool gemeinsam genutzter Datenbankverbindungen noch nicht voll ist. Wenn ein Thread eine Datenbankverbindung schließt, die auch von keinem anderen Thread mehr benutzt wird, wird sie an den Vorratsspeicher inaktiver Datenbankverbindungen zurückgegeben, damit sie wiederverwertet werden kann. Wenn der zugrunde liegende DB-API-Datenbankadapter nicht thread-sicher ist, werden Thread-Locks verwendet, um sicherzustellen, dass die ``pooled_db``-Verbindungen dennoch thread-sicher sind. Sie brauchen sich also hierum keine Sorgen zu machen, aber Sie sollten darauf achten, dedizierte Datenbankverbindungen zu verwenden, sobald Sie Parameter der Datenbanksitzung verändern oder Transaktionen mit mehreren SQL-Befehlen ausführen. Die Qual der Wahl ----------------- Sowohl ``persistent_db`` als auch ``pooled_db`` dienen dem gleichen Zweck, nämlich die Effizienz des Datenbankzugriffs durch Wiederverwendung von Datenbankverbindungen zu steigern, und dabei gleichzeitig die Stabilität zu gewährleisten, selbst wenn die Datenbankverbindung unterbrochen wird. Welches der beiden Module sollte also verwendet werden? Nach den obigen Erklärungen ist es klar, dass ``persistent_db`` dann sinnvoller ist, wenn Ihre Anwendung eine gleich bleibende Anzahl Threads verwendet, die häufig auf die Datenbank zugreifen. In diesem Fall werden Sie ungefähr die gleiche Anzahl geöffneter Datenbankverbindungen erhalten. Wenn jedoch Ihre Anwendung häufig Threads beendet und neu startet, dann ist ``pooled_db`` die bessere Lösung, die auch mehr Möglichkeiten zur Feineinstellung zur Verbesserung der Effizienz erlaubt, insbesondere bei Verwendung eines thread-sicheren DB-API-2-Datenbankadapters. Da die Schnittstellen beider Module sehr ähnlich sind, können Sie recht einfach von einem Modul zum anderen wechseln und austesten, welches geeigneter ist. Benutzung ========= Die Benutzung aller Module ist zwar recht ähnlich, aber es gibt vor allem bei der Initialisierung auch einige Unterschiede, sowohl zwischen den "Pooled"- und den "Persistent"-Varianten, als auch zwischen den DB-API-2- und den PyGreSQL-Varianten. Wir werden hier nur auf das ``persistent_db``-Modul und das etwas kompliziertere ``pooled_db``-Modul eingehen. Einzelheiten zu den anderen Modulen finden Sie in deren Docstrings. Unter Verwendung der Python-Interpreter-Konsole können Sie sich die Dokumentation des ``pooled_db``-Moduls wie folgt anzeigen lassen (dies funktioniert entsprechend auch mit den anderen Modulen):: help(pooled_db) PersistentDB (persistent_db) ---------------------------- Wenn Sie das ``persistent_db``-Modul einsetzen möchten, müssen Sie zuerst einen Generator für die von Ihnen gewünschte Art von Datenbankverbindungen einrichten, indem Sie eine Instanz der Klasse ``PersistentDB`` erzeugen, wobei Sie folgende Parameter angeben müssen: * ``creator``: entweder eine Funktion, die neue DB-API-2-Verbindungen erzeugt, oder ein DB-API-2-Datenbankadapter-Modul * ``maxusage``: Obergrenze dafür, wie oft eine einzelne Verbindung wiederverwendet werden darf (der Standardwert ``0`` oder ``None`` bedeutet unbegrenzte Wiederverwendung) Sobald diese Obergrenze erreicht wird, wird die Verbindung zurückgesetzt. * ``setsession``: eine optionale Liste von SQL-Befehlen zur Initialisierung der Datenbanksitzung, z.B. ``["set datestyle to german", ...]`` * ``failures``: eine optionale Exception-Klasse oder ein Tupel von Exceptions, bei denen die Ausfallsicherung zum Tragen kommen soll, falls die Vorgabe (OperationalError, InterfaceError, InternalError) für das verwendete Datenbankadapter-Modul nicht geeignet sein sollte * ``isfatal``: eine optionale Funktion, der die aufgetretene Exception übergeben wird und die zurückgibt, ob die Ausfallsicherung zum Tragen kommen soll (muss als Schlüsselwort-Argument übergeben werden, siehe `Steuerung der Ausfallsicherung`_) * ``ping``: mit diesem Parameter kann eingestellt werden, wann Verbindungen geprüft werden (``0`` = ``None`` = nie, ``1`` = Standardwert = immer wenn neu angefragt, ``2`` = vor Erzeugen eines Cursors, ``4`` = vor dem Ausführen von Abfragen, ``7`` = immer, und alle Bitkombinationen dieser Werte) Standardmäßig werden Verbindungen mit der ``ping()``-Methode geprüft, falls eine solche vorhanden ist. Sie können für die Prüfung stattdessen auch einen SQL-Befehl oder eine Funktion angeben, die dann wie bei ``ping=1`` erfolgt, oder ein Tupel aus dem obigen Wert und einem der beiden (siehe `Prüfung der Verbindungen`_). * ``closeable``: wenn dies auf ``True`` gesetzt wird, dann wird das Schließen von Verbindungen erlaubt, normalerweise wird es jedoch ignoriert * ``threadlocal``: eine optionale Klasse zur Speicherung thread-lokaler Daten, die anstelle unserer Python-Implementierung benutzt wird (threading.local ist schneller, kann aber nicht in allen Fällen verwendet werden) * Die als ``creator`` angegebene Funktion oder die Funktion ``connect`` des DB-API-2-Datenbankadapter-Moduls erhalten alle weiteren Parameter, wie ``host``, ``database``, ``user``, ``password`` usw. Sie können einige oder alle dieser Parameter in Ihrer eigenen ``creator``-Funktion setzen, was ausgefeilte Mechanismen zur Ausfallsicherung und Lastverteilung ermöglicht. Wenn Sie beispielsweise ``pgdb`` als DB-API-2-Datenbankadapter verwenden, und möchten, dass jede Verbindung Ihrer lokalen Datenbank ``meinedb`` 1000 mal wiederverwendet werden soll, sieht die Initialisierung so aus:: import pgdb # importiere das verwendete DB-API-2-Modul from dbutils.persistent_db import PersistentDB persist = PersistentDB(pgdb, 1000, database='meinedb') Nachdem Sie den Generator mit diesen Parametern eingerichtet haben, können Sie derartige Datenbankverbindungen von da an wie folgt anfordern:: db = persist.connection() Sie können diese Verbindungen verwenden, als wären sie gewöhnliche DB-API-2-Datenbankverbindungen. Genauer genommen erhalten Sie die "gehärtete" ``steady_db``-Version der zugrunde liegenden DB-API-2-Verbindung. Wenn Sie eine solche persistente Verbindung mit ``db.close()`` schließen, wird dies stillschweigend ignoriert, denn sie würde beim nächsten Zugriff sowieso wieder geöffnet, und das wäre nicht im Sinne persistenter Verbindungen. Stattdessen wird die Verbindung automatisch dann geschlossen, wenn der Thread endet. Sie können dieses Verhalten ändern, indem Sie den Parameter namens ``closeable`` setzen. .. warning:: Bitte beachten Sie, dass Transaktionen explizit durch Aufruf der Methode ``begin()`` eingeleitet werden müssen. Hierdurch wird sichergestellt, dass das transparente Neueröffnen von Verbindungen bis zum Ende der Transaktion ausgesetzt wird, und dass die Verbindung zurückgerollt wird, bevor sie vom gleichen Thread erneut benutzt wird. Das Holen einer Verbindung kann etwas beschleunigt werden, indem man den Parameter ``threadlocal`` auf ``threading.local`` setzt; dies könnte aber in einigen Umgebungen nicht funktionieren (es ist zum Beispiel bekannt, dass ``mod_wsgi`` hier Probleme bereitet, da es Daten, die mit ``threading.local`` gespeichert wurden, zwischen Requests löscht). PooledDB (pooled_db) -------------------- Wenn Sie das ``pooled_db``-Modul einsetzen möchten, müssen Sie zuerst einen Pool für die von Ihnen gewünschte Art von Datenbankverbindungen einrichten, indem Sie eine Instanz der Klasse ``PooledDB`` erzeugen, wobei Sie folgende Parameter angeben müssen: * ``creator``: entweder eine Funktion, die neue DB-API-2-Verbindungen erzeugt, oder ein DB-API-2-Datenbankadapter-Modul * ``mincached``: die anfängliche Anzahl inaktiver Verbindungen, die auf Vorrat gehalten werden sollen (der Standardwert ``0`` bedeutet, dass beim Start keine Verbindungen geöffnet werden) * ``maxcached``: Obergrenze für die Anzahl inaktiver Verbindungen, die auf Vorrat gehalten werden sollen (der Standardwert ``0`` oder ``None`` bedeutet unbegrenzte Größe des Vorratsspeichers) * ``maxshared``: Obergrenze für die Anzahl gemeinsam genutzter Verbindungen (der Standardwert ``0`` oder ``None`` bedeutet, dass alle Verbindungen dediziert sind) Wenn diese Obergrenze erreicht wird, werden Verbindungen gemeinsam genutzt, sofern diese als gemeinsam nutzbar angefordert werden. * ``maxconnections``: Obergrenze für die Anzahl an Datenbankverbindungen, die insgesamt überhaupt erlaubt werden sollen (der Standardwert ``0`` oder ``None`` bedeutet unbegrenzte Anzahl von Datenbankverbindungen) * ``blocking``: bestimmt das Verhalten bei Überschreitung dieser Obergrenze Wenn dies auf ``True`` gesetzt wird, dann wird so lange gewartet, bis die Anzahl an Datenbankverbindungen wieder abnimmt, normalerweise wird jedoch sofort eine Fehlermeldung ausgegeben. * ``maxusage``: Obergrenze dafür, wie oft eine einzelne Verbindung wiederverwendet werden darf (der Standardwert ``0`` oder ``None`` bedeutet unbegrenzte Wiederverwendung) Sobald diese Obergrenze erreicht wird, wird die Verbindung automatisch zurückgesetzt (geschlossen und wieder neu geöffnet). * ``setsession``: eine optionale Liste von SQL-Befehlen zur Initialisierung der Datenbanksitzung, z.B. ``["set datestyle to german", ...]`` * ``reset``: wie Verbindungen zurückgesetzt werden sollen, bevor sie wieder in den Verbindungspool zurückgegeben werden (``False`` oder ``None`` um mit ``begin()`` gestartete Transaktionen zurückzurollen, der Standardwert ``True`` rollt sicherheitshalber mögliche Transaktionen immer zurück) * ``failures``: eine optionale Exception-Klasse oder ein Tupel von Exceptions, bei denen die Ausfallsicherung zum Tragen kommen soll, falls die Vorgabe (OperationalError, InterfaceError, InternalError) für das verwendete Datenbankadapter-Modul nicht geeignet sein sollte * ``isfatal``: eine optionale Funktion, der die aufgetretene Exception übergeben wird und die zurückgibt, ob die Ausfallsicherung zum Tragen kommen soll (muss als Schlüsselwort-Argument übergeben werden, siehe `Steuerung der Ausfallsicherung`_) * ``ping``: mit diesem Parameter kann eingestellt werden, wann Verbindungen geprüft werden (``0`` = ``None`` = nie, ``1`` = Standardwert = immer wenn neu angefragt, ``2`` = vor Erzeugen eines Cursors, ``4`` = vor dem Ausführen von Abfragen, ``7`` = immer, und alle Bitkombinationen dieser Werte) Standardmäßig werden Verbindungen mit der ``ping()``-Methode geprüft, falls eine solche vorhanden ist. Sie können für die Prüfung stattdessen auch einen SQL-Befehl oder eine Funktion angeben, die dann wie bei ``ping=1`` erfolgt, oder ein Tupel aus dem obigen Wert und einem der beiden (siehe `Prüfung der Verbindungen`_). * Die als ``creator`` angegebene Funktion oder die Funktion ``connect`` des DB-API-2-Datenbankadapter-Moduls erhalten alle weiteren Parameter, wie ``host``, ``database``, ``user``, ``password`` usw. Sie können einige oder alle dieser Parameter in Ihrer eigenen ``creator``-Funktion setzen, was ausgefeilte Mechanismen zur Ausfallsicherung und Lastverteilung ermöglicht. Wenn Sie beispielsweise ``pgdb`` als DB-API-2-Datenbankadapter benutzen, und einen Pool von mindestens fünf Datenbankverbindungen zu Ihrer Datenbank ``meinedb`` verwenden möchten, dann sieht die Initialisierung so aus:: import pgdb # importiere das verwendete DB-API-2-Modul from dbutils.pooled_db import PooledDB pool = PooledDB(pgdb, 5, database='meinedb') Nachdem Sie den Pool für Datenbankverbindungen so eingerichtet haben, können Sie Verbindungen daraus wie folgt anfordern:: db = pool.connection() Sie können diese Verbindungen verwenden, als wären sie gewöhnliche DB-API-2-Datenbankverbindungen. Genauer genommen erhalten Sie die "gehärtete" ``steady_db``-Version der zugrunde liegenden DB-API-2-Verbindung. Bitte beachten Sie, dass die Verbindung von anderen Threads mitgenutzt werden kann, wenn Sie den Parameter ``maxshared`` auf einen Wert größer als Null gesetzt haben, und der zugrunde liegende DB-API-2-Datenbankadapter dies erlaubt. Eine dedizierte Datenbankverbindung, die garantiert nicht von anderen Threads mitgenutzt wird, fordern Sie wie folgt an:: db = pool.connection(shareable=False) Stattdessen können Sie eine dedizierte Verbindung auch wie folgt erhalten:: db = pool.dedicated_connection() Wenn Sie die Datenbankverbindung nicht mehr benötigen, sollten Sie diese sofort wieder mit ``db.close()`` an den Pool zurückgeben. Sie können auf die gleiche Weise eine neue Verbindung erhalten. .. warning:: In einer Multithread-Umgebung benutzen Sie niemals:: pool.connection().cursor().execute(...) Dies würde die Datenbankverbindung zu früh zur Wiederverwendung zurückgeben, was fatale Folgen haben könnte, wenn die Verbindungen nicht thread-sicher sind. Stellen Sie sicher, dass die Verbindungsobjekte so lange vorhanden sind, wie sie gebraucht werden, etwa so:: db = pool.connection() cur = db.cursor() cur.execute(...) res = cur.fetchone() cur.close() # oder del cur db.close() # oder del db Sie können dies auch durch Verwendung von Kontext-Managern vereinfachen:: with pool.connection() as db: with db.cursor() as cur: cur.execute(...) res = cur.fetchone() .. warning:: Bitte beachten Sie, dass Transaktionen explizit durch Aufruf der Methode ``begin()`` eingeleitet werden müssen. Hierdurch wird sichergestellt, dass die Verbindung nicht mehr mit anderen Threads geteilt wird, dass das transparente Neueröffnen von Verbindungen bis zum Ende der Transaktion ausgesetzt wird, und dass die Verbindung zurückgerollt wird, bevor sie wieder an den Verbindungspool zurückgegeben wird. Besonderheiten bei der Benutzung ================================ Manchmal möchte man Datenbankverbindungen besonders vorbereiten, bevor sie von DBUtils verwendet werden, und dies ist nicht immer durch Verwendung der passenden Parameter möglich. Zum Beispiel kann es ``pyodbc`` erfordern, dass man die Methode ``setencoding()`` der Datenbankverbindung aufruft. Sie können dies erreichen, indem Sie eine modifizierte Version der Funktion ``connect()`` verwenden und diese als ``creator`` (dem ersten Argument) an ``PersistentDB`` oder ``PooledDB`` übergeben, etwa so:: import pyodbc from dbutils.pooled_db import PooledDB def creator(): con = pyodbc.connect(...) con.setencoding(...) return con creator.dbapi = pyodbc db_pool = PooledDB(creator, mincached=5) Steuerung der Ausfallsicherung ------------------------------ Wenn eine Datenbankoperation mit einer der im Parameter ``failures`` angegebenen Exception-Klassen fehlschlägt, dann nimmt DBUtils an, dass die Verbindung verloren gegangen ist, und wiederholt die Operation transparent, zunächst mit einem neuen Cursor und danach mit einer neu geöffneten Verbindung. Genau das möchte man, wenn der Datenbankserver neu gestartet wurde, aber nicht, wenn die Anweisung selbst aus einem Grund fehlgeschlagen ist, der nichts mit dem Zustand der Verbindung zu tun hat. Leider melden viele Datenbankadapter-Module beide Arten von Problemen mit derselben Exception-Klasse. Wenn zum Beispiel MySQL eine Anweisung abbricht, weil diese eine absichtlich angeforderte Zeitbegrenzung überschritten hat:: cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...') dann löst ``pymysql`` einen ``OperationalError`` aus, genau wie beim tatsächlichen Verlust der Verbindung. Da der Parameter ``failures`` nur Exception-*Klassen* angeben kann und Python Unterklassen nicht von einer ``except``-Klausel ausschließen kann, lässt sich diese Unterscheidung nicht durch eine Einschränkung des Parameters ``failures`` ausdrücken. Im obigen Beispiel würde die Anweisung bis zu dreimal ausgeführt, und die Abfrage würde bis zu 30 statt 10 Sekunden dauern. Für solche Fälle können Sie eine Funktion als ``isfatal`` übergeben, die für jeden zu ``failures`` passenden Fehler gefragt wird, ob dieser für die Verbindung wirklich fatal ist:: import pymysql from dbutils.pooled_db import PooledDB def no_timeout(error): # 3024 = Zeitbegrenzung überschritten, die Verbindung ist noch intakt return not error.args or error.args[0] != 3024 pool = PooledDB(pymysql, 5, database='mydb', isfatal=no_timeout) Wenn die Funktion einen falschen Wert zurückgibt, dann gilt der Fehler als nicht fatal, wird also sofort an die Anwendung weitergegeben, und die Verbindung wird weder ersetzt noch aus dem Verbindungspool entfernt. Beachten Sie, dass ``isfatal`` als Schlüsselwort-Argument übergeben werden muss und mit beliebigen Exceptions der als ``failures`` angegebenen Klassen umgehen können muss. Wenn Sie die Ausfallsicherung nur für eine bestimmte Anweisung abschalten möchten, statt sie für den gesamten Verbindungspool zu konfigurieren, dann können Sie den Kontextmanager ``no_failover()`` des Cursors verwenden:: with pool.connection() as db: with db.cursor() as cur: with cur.no_failover(): cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...') rows = cur.fetchall() Alles Übrige, insbesondere die ``ping()``-Prüfung und die Zählung der Verwendungen der zugrunde liegenden Verbindung, bleibt innerhalb dieses Kontexts unverändert. Als letztes Mittel können Sie über das Attribut ``dbapi_connection`` einer Verbindung und das Attribut ``dbapi_cursor`` eines Cursors auch direkt auf die zugrunde liegenden DB-API-2-Objekte zugreifen. Beachten Sie, dass Operationen auf diesen Objekten überhaupt nicht von der Ausfallsicherung erfasst und auch nicht auf die Begrenzung ``maxusage`` angerechnet werden. .. warning:: Statt der obigen Möglichkeiten können Sie auch einfach mit der Methode ``begin()`` eine Transaktion starten, da die Ausfallsicherung während einer Transaktion stets ausgesetzt wird. Bedenken Sie aber, dass dadurch tatsächlich eine Transaktion auf dem Datenbankserver gestartet wird, mit allen sich daraus ergebenden Folgen. Prüfung der Verbindungen ------------------------ Verbindungen, die im Verbindungspool gehalten oder einem Thread zugeordnet sind, können unbrauchbar werden, wenn der Datenbankserver neu gestartet wird oder zu lange untätige Verbindungen abbricht. Damit dies auffällt, bevor die Verbindung verwendet wird, kann DBUtils sie mit der ``ping()``-Methode der zugrunde liegenden DB-API-2-Verbindung prüfen, wobei der Parameter ``ping`` festlegt, wann eine solche Prüfung stattfindet. Allerdings ist ``ping()`` nicht Teil der DB-API-2-Spezifikation, und viele gebräuchliche Datenbankadapter-Module stellen diese Methode nicht bereit. Sobald DBUtils bemerkt, dass die Methode nicht verfügbar ist, werden die Verbindungen überhaupt nicht mehr geprüft. In diesem Fall können Sie als Parameter ``ping`` einen SQL-Befehl angeben, der stattdessen zur Prüfung der Verbindung ausgeführt werden soll:: import psycopg2 from dbutils.pooled_db import PooledDB pool = PooledDB(psycopg2, 5, database='meinedb', ping='select 1') Statt eines SQL-Befehls können Sie auch eine Funktion angeben, der die zugrunde liegende DB-API-2-Verbindung übergeben wird und die zurückgibt, ob diese noch intakt ist:: def is_healthy(con): # psycopg2 weiß, ob die Verbindung bereits geschlossen wurde return not con.closed pool = PooledDB(psycopg2, 5, database='meinedb', ping=is_healthy) Die Rückgabe von ``None`` gilt als intakt, das Auslösen einer Exception als nicht intakt, so dass auch eine Funktion verwendet werden kann, die einfach eine Datenbankoperation ausführt, ohne Fehler abzufangen. Meldet die Prüfung die Verbindung als nicht intakt, dann wird diese transparent durch eine neu geöffnete Verbindung ersetzt, genau wie wenn ``ping()`` sie als nicht mehr brauchbar gemeldet hätte. Eine solche Prüfung findet immer dann statt, wenn eine Verbindung aus dem Verbindungspool geholt oder von einem Thread angefragt wird, also zu dem Zeitpunkt, der dem Wert ``ping=1`` entspricht. Wenn sie auch zu den anderen Zeitpunkten stattfinden soll, dann geben Sie ein Tupel aus dem üblichen ganzzahligen Wert und dem SQL-Befehl oder der Funktion an:: pool = PooledDB(psycopg2, 5, database='meinedb', ping=(4, 'select 1')) .. warning:: Jede derartige Prüfung erfordert eine zusätzliche Anfrage an den Datenbankserver, weshalb Sie sie nur bei wirklichem Bedarf mit ``ping=4`` oder ``ping=7`` kombinieren sollten. Bedenken Sie außerdem, dass ein zur Prüfung ausgeführter SQL-Befehl eine Transaktion auf dem Datenbankserver starten kann, wenn die Verbindung nicht im Autocommit-Modus läuft. Anmerkungen =========== Wenn Sie einen der bekannten "Object-Relational Mapper" SQLObject_ oder SQLAlchemy_ verwenden, dann benötigen Sie DBUtils nicht, denn diese haben ihre eigenen Mechanismen zum Pooling von Datenbankverbindungen eingebaut. Tatsächlich hat SQLObject 2 (SQL-API) das Pooling in eine separate Schicht ausgelagert, in der Code von DBUtils verwendet wird. Wenn Sie eine Lösung verwenden wie den Apache-Webserver mit mod_python_ oder mod_wsgi_, dann sollten Sie bedenken, dass Ihr Python-Code normalerweise im Kontext der Kindprozesse des Webservers läuft. Wenn Sie also das ``pooled_db``-Modul einsetzen, und mehrere dieser Kindprozesse laufen, dann werden Sie ebenso viele Pools mit Datenbankverbindungen erhalten. Wenn diese Prozesse viele Threads laufen lassen, dann mag dies ein sinnvoller Ansatz sein, wenn aber diese Prozesse nicht mehr als einen Worker-Thread starten, wie im Fall des Multi-Processing-Moduls "prefork" für den Apache-Webserver, dann sollten Sie auf eine Middleware für das Connection-Pooling zurückgreifen, die Multi-Processing unterstützt, wie zum Beispiel pgpool_ oder pgbouncer_ für die PostgreSQL-Datenbank. Zukunft ======= Einige Ideen für zukünftige Verbesserungen: * Alternativ zur Obergrenze in der Anzahl der Nutzung einer Datenbankverbindung könnte eine maximale Lebensdauer für die Verbindung implementiert werden. * Es könnten Module ``monitor_db`` und ``monitor_pg`` hinzugefügt werden, die in einem separaten Thread ständig den "idle pool" und eventuell auch den "shared pool" bzw. die persistenten Verbindungen überwachen. Wenn eine unterbrochene Datenbankverbindung entdeckt wird, wird diese automatisch durch den Monitor-Thread wiederhergestellt. Dies ist in einem Szenario sinnvoll, bei dem die Datenbank einer Website jede Nacht neu gestartet wird. Ohne den Monitor-Thread würden die Benutzer morgens eine kleine Verzögerung bemerken, weil erst dann die unterbrochenen Datenbankverbindungen entdeckt würden und sich der Pool langsam wieder neu aufbaut. Mit dem Monitor-Thread würde dies schon während der Nacht passieren, kurz nach der Unterbrechung. Der Monitor-Thread könnte auch so konfiguriert werden, dass er generell täglich den Verbindungspool erneuert, kurz bevor die Benutzer erscheinen. * Optional sollten Benutzung, schlechte Verbindungen und Überschreitung von Obergrenzen in Logs gespeichert werden können. Fehlermeldungen und Feedback ============================ Fehlermeldungen, Patches und Feedback können Sie als Issues_ oder `Pull Requests`_ auf der `GitHub-Projektseite`_ von DBUtils übermitteln. .. _GitHub-Projektseite: https://github.com/WebwareForPython/DBUtils .. _Issues: https://github.com/WebwareForPython/DBUtils/issues .. _Pull Requests: https://github.com/WebwareForPython/DBUtils/pulls Links ===== Einige Links zu verwandter und alternativer Software: * DBUtils_ * Python_ * `Webware for Python`_ Framework * Python `DB-API 2`_ * PostgreSQL_ Datenbank * PyGreSQL_ Python-Adapter für PostgreSQL * pgpool_ Middleware für Connection-Pooling mit PostgreSQL * pgbouncer_ Middleware für Connection-Pooling mit PostgreSQL * SQLObject_ Objekt-relationaler Mapper * SQLAlchemy_ Objekt-relationaler Mapper .. _DBUtils: https://github.com/WebwareForPython/DBUtils .. _Python: https://www.python.org .. _Webware for Python: https://webwareforpython.github.io/w4py/ .. _DB-API 2: https://www.python.org/dev/peps/pep-0249/ .. _The Python DB-API: http://www.linuxjournal.com/article/2605 .. _PostgresQL: https://www.postgresql.org/ .. _PyGreSQL: https://www.pygresql.org/ .. _SQLObject: http://sqlobject.org/ .. _SQLAlchemy: https://www.sqlalchemy.org .. _Apache: https://httpd.apache.org/ .. _mod_python: http://modpython.org/ .. _mod_wsgi: https://github.com/GrahamDumpleton/mod_wsgi .. _pgpool: https://www.pgpool.net/ .. _pgbouncer: https://pgbouncer.github.io/ Autoren ======= :Autor: `Christoph Zwerschke`_ :Beiträge: DBUtils benutzt Code, Anmerkungen und Vorschläge von Ian Bicking, Chuck Esterbrook (Webware for Python), Dan Green (DBTools), Jay Love, Michael Palmer, Tom Schwaller, Geoffrey Talvola, Warren Smith (DbConnectionPool), Ezio Vernacotola, Jehiah Czebotar, Matthew Harriger, Gregory Piñero und Josef van Eenbergen. .. _Christoph Zwerschke: https://github.com/Cito Copyright und Lizenz ==================== Copyright © 2005-2025 Christoph Zwerschke. Alle Rechte vorbehalten. DBUtils ist freie und quelloffene Software, lizenziert unter der `MIT-Lizenz`__. __ https://opensource.org/licenses/MIT WebwareForPython-DBUtils-8de754c/docs/main.html000066400000000000000000001206411524214146500214620ustar00rootroot00000000000000 DBUtils User's Guide

DBUtils User's Guide

Version:
3.2.0
Translations:

English | German

Synopsis

DBUtils is a suite of Python modules that allows you to connect in a safe and efficient way between a threaded Python application and a database.

DBUtils has been originally written particularly for Webware for Python as the application and PyGreSQL as the adapter to a PostgreSQL database, but it can meanwhile be used for any other Python application and DB-API 2 conformant database adapter.

Modules

The DBUtils suite is realized as a Python package containing two subsets of modules, one for use with arbitrary DB-API 2 modules, the other one for use with the classic PyGreSQL module.

Universal DB-API 2 variant

steady_db

Hardened DB-API 2 connections

pooled_db

Pooling for DB-API 2 connections

persistent_db

Persistent DB-API 2 connections

simple_pooled_db

Simple pooling for DB-API 2

Classic PyGreSQL variant

steady_pg

Hardened classic PyGreSQL connections

pooled_pg

Pooling for classic PyGreSQL connections

persistent_pg

Persistent classic PyGreSQL connections

simple_pooled_pg

Simple pooling for classic PyGreSQL

The dependencies of the modules in the universal DB-API 2 variant are as indicated in the following diagram:

dependencies_db.png

The dependencies of the modules in the classic PyGreSQL variant are similar:

dependencies_pg.png

Download

You can download the current version of DBUtils from the Python Package Index at:

https://pypi.python.org/pypi/DBUtils

The source code repository can be found here on GitHub:

https://github.com/WebwareForPython/DBUtils

Installation

The package can be installed in the usual way:

python setup.py install

It is even easier to download and install the package in one go using pip:

pip install DBUtils

Requirements

DBUtils supports Python versions 3.7 to 3.14.

The modules in the classic PyGreSQL variant need PyGreSQL version 4.0 or above, while the modules in the universal DB-API 2 variant run with any Python DB-API 2 compliant database interface module.

Functionality

This section will refer to the names in the DB-API 2 variant only, but the same applies to the classic PyGreSQL variant.

DBUtils installs itself as a package dbutils containing all the modules that are described in this guide. Each of these modules contains essentially one class with an analogous name that provides the corresponding functionality. For instance, the module dbutils.pooled_db contains the class PooledDB.

SimplePooledDB (simple_pooled_db)

The class SimplePooledDB in dbutils.simple_pooled_db is a very basic reference implementation of a pooled database connection. It is much less sophisticated than the regular pooled_db module and is particularly lacking the failover functionality. dbutils.simple_pooled_db is essentially the same as the MiscUtils.DBPool module that is part of Webware for Python. You should consider it a demonstration of concept rather than something that should go into production.

SteadyDBConnection (steady_db)

The class SteadyDBConnection in the module dbutils.steady_db implements "hardened" connections to a database, based on ordinary connections made by any DB-API 2 database module. A "hardened" connection will transparently reopen upon access when it has been closed or the database connection has been lost or when it is used more often than an optional usage limit.

A typical example where this is needed is when the database has been restarted while your application is still running and has open connections to the database, or when your application accesses a remote database in a network that is separated by a firewall and the firewall has been restarted and lost its state.

Usually, you will not use the steady_db module directly; it merely serves as a basis for the next two modules, persistent_db and pooled_db.

PersistentDB (persistent_db)

The class PersistentDB in the module dbutils.persistent_db implements steady, thread-affine, persistent connections to a database, using any DB-API 2 database module. "Thread-affine" and "persistent" means that the individual database connections stay assigned to the respective threads and will not be closed during the lifetime of the threads.

The following diagram shows the connection layers involved when you are using persistent_db connections:

persistent.png

Whenever a thread opens a database connection for the first time, a new connection to the database will be opened that will be used from now on for this specific thread. When the thread closes the database connection, it will still be kept open so that the next time when a connection is requested by the same thread, this already opened connection can be used. The connection will be closed automatically when the thread dies.

In short: persistent_db tries to recycle database connections to increase the overall database access performance of your threaded application, but it makes sure that connections are never shared between threads.

Therefore, persistent_db will work perfectly even if the underlying DB-API module is not thread-safe at the connection level, and it will avoid problems when other threads change the database session or perform transactions spreading over more than one SQL command.

PooledDB (pooled_db)

The class PooledDB in the module dbutils.pooled_db implements a pool of steady, thread-safe cached connections to a database which are transparently reused, using any DB-API 2 database module.

The following diagram shows the connection layers involved when you are using pooled_db connections:

pooled.png

As the diagram indicates, pooled_db can share opened database connections between different threads. This will happen by default if you set up the connection pool with a positive value of maxshared and the underlying DB-API 2 is thread-safe at the connection level, but you can also request dedicated database connections that will not be shared between threads. Besides the pool of shared connections, you can also set up a pool of at least mincached and at most maxcached idle connections that will be used whenever a thread is requesting a dedicated database connection or the pool of shared connections is not yet full. When a thread closes a connection that is not shared anymore, it is returned back to the pool of idle connections so that it can be recycled again.

If the underlying DB-API module is not thread-safe, thread locks will be used to ensure that the pooled_db connections are thread-safe. So you don't need to worry about that, but you should be careful to use dedicated connections whenever you change the database session or perform transactions spreading over more than one SQL command.

Which one to use?

Both persistent_db and pooled_db serve the same purpose to improve the database access performance by recycling database connections, while preserving stability even if the database connection is disrupted.

So which of these two modules should you use? From the above explanations it is clear that persistent_db will make more sense if your application keeps a constant number of threads which frequently use the database. In this case, you will always have the same amount of open database connections. However, if your application frequently starts and ends threads, then it will be better to use pooled_db. The latter will also allow more fine-tuning, particularly if you are using a thread-safe DB-API 2 module.

Since the interface of both modules is similar, you can easily switch from one to the other and check which one will suit better.

Usage

The usage of all the modules is similar, but there are also some differences in the initialization between the "Pooled" and "Persistent" variants and also between the universal DB-API 2 and the classic PyGreSQL variants.

We will cover here only the persistent_db module and the more complex pooled_db module. For the details of the other modules, have a look at their module docstrings. Using the Python interpreter console, you can display the documentation of the pooled_db module as follows (this works analogously for the other modules):

help(pooled_db)

PersistentDB (persistent_db)

In order to make use of the persistent_db module, you first need to set up a generator for your kind of database connections by creating an instance of PersistentDB, passing the following parameters:

  • creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module

  • maxusage: the maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse)

    Whenever the limit is reached, the connection will be reset.

  • setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...]

  • failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module

  • isfatal: an optional callable that is passed the exception which has been raised and shall return whether the connection failover mechanism shall be applied (must be passed as a keyword argument, see Controlling the failover mechanism)

  • ping: an optional flag controlling when connections are checked (0 = None = never, 1 = default = whenever it is requested, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values)

    By default, connections are checked with the ping() method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ping=1, or a tuple with the flag and one of the latter (see Checking connections).

  • closeable: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored

  • threadlocal: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases)

  • The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms.

For instance, if you are using pgdb as your DB-API 2 database module and want every connection to your local database mydb to be reused 1000 times:

import pgdb  # import used DB-API 2 module
from dbutils.persistent_db import PersistentDB
persist = PersistentDB(pgdb, 1000, database='mydb')

Once you have set up the generator with these parameters, you can request database connections of that kind:

db = persist.connection()

You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened steady_db version of the underlying DB-API 2 connection.

Closing a persistent connection with db.close() will be silently ignored since it would be reopened at the next usage anyway and contrary to the intent of having persistent connections. Instead, the connection will be automatically closed when the thread dies. You can change this behavior by setting the closeable parameter.

By setting the threadlocal parameter to threading.local, getting connections may become a bit faster, but this may not work in all environments (for instance, mod_wsgi is known to cause problems since it clears the threading.local data between requests).

PooledDB (pooled_db)

In order to make use of the pooled_db module, you first need to set up the database connection pool by creating an instance of PooledDB, passing the following parameters:

  • creator: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module

  • mincached: the initial number of idle connections in the pool (the default of 0 means no connections are made at startup)

  • maxcached: the maximum number of idle connections in the pool (the default value of 0 or None means unlimited pool size)

  • maxshared: maximum number of shared connections allowed (the default value of 0 or None means all connections are dedicated)

    When this maximum number is reached, connections are shared if they have been requested as shareable.

  • maxconnections: maximum number of connections generally allowed (the default value of 0 or None means any number of connections)

  • blocking: determines behavior when exceeding the maximum

    If this is set to true, block and wait until the number of connections decreases, but by default an error will be reported.

  • maxusage: maximum number of reuses of a single connection (the default of 0 or None means unlimited reuse)

    When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened).

  • setsession: an optional list of SQL commands that may serve to prepare the session, e.g. ["set datestyle to german", ...]

  • reset: how connections should be reset when returned to the pool (False or None to rollback transactions started with begin(), the default value True always issues a rollback for safety's sake)

  • failures: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module

  • isfatal: an optional callable that is passed the exception which has been raised and shall return whether the connection failover mechanism shall be applied (must be passed as a keyword argument, see Controlling the failover mechanism)

  • ping: an optional flag controlling when connections are checked (0 = None = never, 1 = default = whenever fetched from the pool, 2 = when a cursor is created, 4 = when a query is executed, 7 = always, and all other bit combinations of these values)

    By default, connections are checked with the ping() method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ping=1, or a tuple with the flag and one of the latter (see Checking connections).

  • The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms.

For instance, if you are using pgdb as your DB-API 2 database module and want a pool of at least five connections to your local database mydb:

import pgdb  # import used DB-API 2 module
from dbutils.pooled_db import PooledDB
pool = PooledDB(pgdb, 5, database='mydb')

Once you have set up the connection pool you can request database connections from that pool:

db = pool.connection()

You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened steady_db version of the underlying DB-API 2 connection.

Please note that the connection may be shared with other threads by default if you set a non-zero maxshared parameter and the DB-API 2 module allows this. If you want to have a dedicated connection, use:

db = pool.connection(shareable=False)

Instead of this, you can also get a dedicated connection as follows:

db = pool.dedicated_connection()

If you don't need it anymore, you should immediately return it to the pool with db.close(). You can get another connection in the same way.

You can also use context managers for simpler code:

with pool.connection() as db:
    with db.cursor() as cur:
        cur.execute(...)
        res = cur.fetchone()

Advanced Usage

Sometimes you may want to prepare connections before they are used by DBUtils, in ways that are not possible by just using the right parameters. For instance, pyodbc may require to configure connections by calling the setencoding() method of the connection. You can do this by passing a modified connect() function to PersistentDB or PooledDB as creator (the first argument), like this:

import pyodbc
from dbutils.pooled_db import PooledDB

def creator():
    con = pyodbc.connect(...)
    con.setencoding(...)
    return con

creator.dbapi = pyodbc

db_pool = PooledDB(creator, mincached=5)

Controlling the failover mechanism

When a database operation fails with one of the exception classes listed in the failures parameter, DBUtils assumes that the connection has been lost and transparently retries the operation, first with a new cursor, and then with a newly opened connection. This is exactly what you want when the database server has been restarted, but not when the statement itself failed for a reason that has nothing to do with the state of the connection.

Unfortunately, many database modules report both kinds of problems with the same exception class. For instance, when MySQL aborts a statement because it exceeded a deliberately requested time limit:

cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...')

then pymysql raises an OperationalError, just as it would when the connection has really been lost. Since the failures parameter can only specify exception classes, and Python cannot exclude subclasses from an except clause, you cannot express this distinction by narrowing the failures parameter. In the above example, the statement would be executed up to three times, and the query would take up to 30 instead of 10 seconds.

For such cases you can pass an isfatal callable, which is asked for every error matching failures whether that error is really fatal for the connection:

import pymysql
from dbutils.pooled_db import PooledDB

def no_timeout(error):
    # 3024 = "max execution time exceeded", the connection is still fine
    return not error.args or error.args[0] != 3024

pool = PooledDB(pymysql, 5, database='mydb', isfatal=no_timeout)

When the callable returns a false value, the error is not considered fatal, so it is raised to the application immediately, and the connection is neither replaced nor removed from the pool. Note that isfatal must be passed as a keyword argument, and that it must tolerate arbitrary exception instances of the classes given as failures.

If you only want to suppress the failover mechanism for a particular statement instead of configuring it for the whole pool, you can use the no_failover() context manager of the cursor:

with pool.connection() as db:
    with db.cursor() as cur:
        with cur.no_failover():
            cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...')
        rows = cur.fetchall()

Everything else, particularly the ping check and the usage bookkeeping of the underlying steady connection, is left untouched inside that context.

As a last resort, you can also reach through to the underlying DB-API 2 objects with the dbapi_connection attribute of a connection and the dbapi_cursor attribute of a cursor. Note that operations executed on these objects are not covered by the failover mechanism at all, and that they are not counted towards the maxusage limit either.

Checking connections

Connections that are kept in the pool or bound to a thread may become stale when the database server is restarted or drops connections that have been idle for too long. In order to notice this before the connection is used, DBUtils can check it with the ping() method of the underlying DB-API 2 connection, and the ping parameter controls when such a check is made.

However, ping() is not part of the DB-API 2 specification, and many widely used database modules do not implement it. When DBUtils notices that the method is unavailable, it stops checking the connections altogether. In this case, you can pass an SQL statement as ping parameter that shall be executed for checking the connection instead:

import psycopg2
from dbutils.pooled_db import PooledDB

pool = PooledDB(psycopg2, 5, database='mydb', ping='select 1')

Instead of an SQL statement, you can also pass a callable that is passed the underlying DB-API 2 connection and shall return whether it is still alive:

def is_healthy(con):
    # psycopg2 knows whether the connection has been closed already
    return not con.closed

pool = PooledDB(psycopg2, 5, database='mydb', ping=is_healthy)

Returning None counts as alive and raising an exception counts as not alive, so a callable that simply carries out some database operation without catching errors can be used as well. When the check reports the connection as not alive, it is transparently replaced with a newly opened connection, just as if ping() had reported it as broken.

Such a check is made whenever a connection is fetched from the pool or requested by a thread, i.e. at the time corresponding to ping=1. If you want it to be made at the other times as well, pass a tuple with the usual integer value and the SQL statement or the callable:

pool = PooledDB(psycopg2, 5, database='mydb', ping=(4, 'select 1'))

Notes

If you are using one of the popular object-relational mappers SQLObject or SQLAlchemy, you won't need DBUtils, since they come with their own connection pools. SQLObject 2 (SQL-API) is actually borrowing some code from DBUtils to split the pooling out into a separate layer.

Also note that when you are using a solution like the Apache webserver with mod_python or mod_wsgi, then your Python code will be usually run in the context of the webserver's child processes. So if you are using the pooled_db module, and several of these child processes are running, you will have as many database connection pools. If these processes are running many threads, this may still be a reasonable approach, but if these processes don't spawn more than one worker thread, as in the case of Apache's "prefork" multi-processing module, this approach does not make sense. If you're running such a configuration, you should resort to a middleware for connection pooling that supports multi-processing, such as pgpool or pgbouncer for the PostgreSQL database.

Future

Some ideas for future improvements:

  • Alternatively to the maximum number of uses of a connection, implement a maximum time to live for connections.

  • Create modules monitor_db and monitor_pg that will run in a separate thread, monitoring the pool of the idle connections and maybe also the shared connections respectively the thread-affine connections. If a disrupted connection is detected, then it will be reestablished automatically by the monitoring thread. This will be useful in a scenario where a database powering a website is restarted during the night. Without the monitoring thread, the users would experience a slight delay in the next morning, because only then, the disrupted database connections will be detected and the pool will be rebuilt. With the monitoring thread, this will already happen during the night, shortly after the disruption. The monitoring thread could also be configured to generally recreate the connection pool every day shortly before the users arrive.

  • Optionally log usage, bad connections and exceeding of limits.

Bug reports and feedback

You can transmit bug reports, patches and feedback by creating issues or pull requests on the GitHub project page for DBUtils.

Credits

Author:

Christoph Zwerschke

Contributions:

DBUtils uses code, input and suggestions made by Ian Bicking, Chuck Esterbrook (Webware for Python), Dan Green (DBTools), Jay Love, Michael Palmer, Tom Schwaller, Geoffrey Talvola, Warren Smith (DbConnectionPool), Ezio Vernacotola, Jehiah Czebotar, Matthew Harriger, Gregory Piñero and Josef van Eenbergen.

WebwareForPython-DBUtils-8de754c/docs/main.rst000066400000000000000000000716161524214146500213350ustar00rootroot00000000000000DBUtils User's Guide ++++++++++++++++++++ :Version: 3.2.0 :Translations: English | German_ .. _German: main.de.html .. contents:: Contents Synopsis ======== DBUtils_ is a suite of Python modules that allows you to connect in a safe and efficient way between a threaded Python_ application and a database. DBUtils has been originally written particularly for `Webware for Python`_ as the application and PyGreSQL_ as the adapter to a PostgreSQL_ database, but it can meanwhile be used for any other Python application and `DB-API 2`_ conformant database adapter. Modules ======= The DBUtils suite is realized as a Python package containing two subsets of modules, one for use with arbitrary DB-API 2 modules, the other one for use with the classic PyGreSQL module. +------------------+------------------------------------------+ | Universal DB-API 2 variant | +==================+==========================================+ | steady_db | Hardened DB-API 2 connections | +------------------+------------------------------------------+ | pooled_db | Pooling for DB-API 2 connections | +------------------+------------------------------------------+ | persistent_db | Persistent DB-API 2 connections | +------------------+------------------------------------------+ | simple_pooled_db | Simple pooling for DB-API 2 | +------------------+------------------------------------------+ +------------------+------------------------------------------+ | Classic PyGreSQL variant | +==================+==========================================+ | steady_pg | Hardened classic PyGreSQL connections | +------------------+------------------------------------------+ | pooled_pg | Pooling for classic PyGreSQL connections | +------------------+------------------------------------------+ | persistent_pg | Persistent classic PyGreSQL connections | +------------------+------------------------------------------+ | simple_pooled_pg | Simple pooling for classic PyGreSQL | +------------------+------------------------------------------+ The dependencies of the modules in the universal DB-API 2 variant are as indicated in the following diagram: .. image:: dependencies_db.png The dependencies of the modules in the classic PyGreSQL variant are similar: .. image:: dependencies_pg.png Download ======== You can download the current version of DBUtils from the Python Package Index at:: https://pypi.python.org/pypi/DBUtils The source code repository can be found here on GitHub:: https://github.com/WebwareForPython/DBUtils Installation ============ The package can be installed in the usual way:: python setup.py install It is even easier to download and install the package in one go using `pip`_:: pip install DBUtils .. _pip: https://pip.pypa.io/ Requirements ============ DBUtils supports Python_ versions 3.7 to 3.14. The modules in the classic PyGreSQL variant need PyGreSQL_ version 4.0 or above, while the modules in the universal DB-API 2 variant run with any Python `DB-API 2`_ compliant database interface module. Functionality ============= This section will refer to the names in the DB-API 2 variant only, but the same applies to the classic PyGreSQL variant. DBUtils installs itself as a package ``dbutils`` containing all the modules that are described in this guide. Each of these modules contains essentially one class with an analogous name that provides the corresponding functionality. For instance, the module ``dbutils.pooled_db`` contains the class ``PooledDB``. SimplePooledDB (simple_pooled_db) --------------------------------- The class ``SimplePooledDB`` in ``dbutils.simple_pooled_db`` is a very basic reference implementation of a pooled database connection. It is much less sophisticated than the regular ``pooled_db`` module and is particularly lacking the failover functionality. ``dbutils.simple_pooled_db`` is essentially the same as the ``MiscUtils.DBPool`` module that is part of Webware for Python. You should consider it a demonstration of concept rather than something that should go into production. SteadyDBConnection (steady_db) ------------------------------ The class ``SteadyDBConnection`` in the module ``dbutils.steady_db`` implements "hardened" connections to a database, based on ordinary connections made by any DB-API 2 database module. A "hardened" connection will transparently reopen upon access when it has been closed or the database connection has been lost or when it is used more often than an optional usage limit. A typical example where this is needed is when the database has been restarted while your application is still running and has open connections to the database, or when your application accesses a remote database in a network that is separated by a firewall and the firewall has been restarted and lost its state. Usually, you will not use the ``steady_db`` module directly; it merely serves as a basis for the next two modules, ``persistent_db`` and ``pooled_db``. PersistentDB (persistent_db) ---------------------------- The class ``PersistentDB`` in the module ``dbutils.persistent_db`` implements steady, thread-affine, persistent connections to a database, using any DB-API 2 database module. "Thread-affine" and "persistent" means that the individual database connections stay assigned to the respective threads and will not be closed during the lifetime of the threads. The following diagram shows the connection layers involved when you are using ``persistent_db`` connections: .. image:: persistent.png Whenever a thread opens a database connection for the first time, a new connection to the database will be opened that will be used from now on for this specific thread. When the thread closes the database connection, it will still be kept open so that the next time when a connection is requested by the same thread, this already opened connection can be used. The connection will be closed automatically when the thread dies. In short: ``persistent_db`` tries to recycle database connections to increase the overall database access performance of your threaded application, but it makes sure that connections are never shared between threads. Therefore, ``persistent_db`` will work perfectly even if the underlying DB-API module is not thread-safe at the connection level, and it will avoid problems when other threads change the database session or perform transactions spreading over more than one SQL command. PooledDB (pooled_db) -------------------- The class ``PooledDB`` in the module ``dbutils.pooled_db`` implements a pool of steady, thread-safe cached connections to a database which are transparently reused, using any DB-API 2 database module. The following diagram shows the connection layers involved when you are using ``pooled_db`` connections: .. image:: pooled.png As the diagram indicates, ``pooled_db`` can share opened database connections between different threads. This will happen by default if you set up the connection pool with a positive value of ``maxshared`` and the underlying DB-API 2 is thread-safe at the connection level, but you can also request dedicated database connections that will not be shared between threads. Besides the pool of shared connections, you can also set up a pool of at least ``mincached`` and at most ``maxcached`` idle connections that will be used whenever a thread is requesting a dedicated database connection or the pool of shared connections is not yet full. When a thread closes a connection that is not shared anymore, it is returned back to the pool of idle connections so that it can be recycled again. If the underlying DB-API module is not thread-safe, thread locks will be used to ensure that the ``pooled_db`` connections are thread-safe. So you don't need to worry about that, but you should be careful to use dedicated connections whenever you change the database session or perform transactions spreading over more than one SQL command. Which one to use? ----------------- Both ``persistent_db`` and ``pooled_db`` serve the same purpose to improve the database access performance by recycling database connections, while preserving stability even if the database connection is disrupted. So which of these two modules should you use? From the above explanations it is clear that ``persistent_db`` will make more sense if your application keeps a constant number of threads which frequently use the database. In this case, you will always have the same amount of open database connections. However, if your application frequently starts and ends threads, then it will be better to use ``pooled_db``. The latter will also allow more fine-tuning, particularly if you are using a thread-safe DB-API 2 module. Since the interface of both modules is similar, you can easily switch from one to the other and check which one will suit better. Usage ===== The usage of all the modules is similar, but there are also some differences in the initialization between the "Pooled" and "Persistent" variants and also between the universal DB-API 2 and the classic PyGreSQL variants. We will cover here only the ``persistent_db`` module and the more complex ``pooled_db`` module. For the details of the other modules, have a look at their module docstrings. Using the Python interpreter console, you can display the documentation of the ``pooled_db`` module as follows (this works analogously for the other modules):: help(pooled_db) PersistentDB (persistent_db) ---------------------------- In order to make use of the ``persistent_db`` module, you first need to set up a generator for your kind of database connections by creating an instance of ``PersistentDB``, passing the following parameters: * ``creator``: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module * ``maxusage``: the maximum number of reuses of a single connection (the default of ``0`` or ``None`` means unlimited reuse) Whenever the limit is reached, the connection will be reset. * ``setsession``: an optional list of SQL commands that may serve to prepare the session, e.g. ``["set datestyle to german", ...]`` * ``failures``: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module * ``isfatal``: an optional callable that is passed the exception which has been raised and shall return whether the connection failover mechanism shall be applied (must be passed as a keyword argument, see `Controlling the failover mechanism`_) * ``ping``: an optional flag controlling when connections are checked (``0`` = ``None`` = never, ``1`` = default = whenever it is requested, ``2`` = when a cursor is created, ``4`` = when a query is executed, ``7`` = always, and all other bit combinations of these values) By default, connections are checked with the ``ping()`` method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ``ping=1``, or a tuple with the flag and one of the latter (see `Checking connections`_). * ``closeable``: if this is set to true, then closing connections will be allowed, but by default this will be silently ignored * ``threadlocal``: an optional class for representing thread-local data that will be used instead of our Python implementation (threading.local is faster, but cannot be used in all cases) * The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms. For instance, if you are using ``pgdb`` as your DB-API 2 database module and want every connection to your local database ``mydb`` to be reused 1000 times:: import pgdb # import used DB-API 2 module from dbutils.persistent_db import PersistentDB persist = PersistentDB(pgdb, 1000, database='mydb') Once you have set up the generator with these parameters, you can request database connections of that kind:: db = persist.connection() You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened ``steady_db`` version of the underlying DB-API 2 connection. Closing a persistent connection with ``db.close()`` will be silently ignored since it would be reopened at the next usage anyway and contrary to the intent of having persistent connections. Instead, the connection will be automatically closed when the thread dies. You can change this behavior by setting the ``closeable`` parameter. .. warning:: Note that you need to explicitly start transactions by calling the ``begin()`` method. This ensures that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being reused by the same thread. By setting the ``threadlocal`` parameter to ``threading.local``, getting connections may become a bit faster, but this may not work in all environments (for instance, ``mod_wsgi`` is known to cause problems since it clears the ``threading.local`` data between requests). PooledDB (pooled_db) -------------------- In order to make use of the ``pooled_db`` module, you first need to set up the database connection pool by creating an instance of ``PooledDB``, passing the following parameters: * ``creator``: either an arbitrary function returning new DB-API 2 connection objects or a DB-API 2 compliant database module * ``mincached``: the initial number of idle connections in the pool (the default of ``0`` means no connections are made at startup) * ``maxcached``: the maximum number of idle connections in the pool (the default value of ``0`` or ``None`` means unlimited pool size) * ``maxshared``: maximum number of shared connections allowed (the default value of ``0`` or ``None`` means all connections are dedicated) When this maximum number is reached, connections are shared if they have been requested as shareable. * ``maxconnections``: maximum number of connections generally allowed (the default value of ``0`` or ``None`` means any number of connections) * ``blocking``: determines behavior when exceeding the maximum If this is set to true, block and wait until the number of connections decreases, but by default an error will be reported. * ``maxusage``: maximum number of reuses of a single connection (the default of ``0`` or ``None`` means unlimited reuse) When this maximum usage number of the connection is reached, the connection is automatically reset (closed and reopened). * ``setsession``: an optional list of SQL commands that may serve to prepare the session, e.g. ``["set datestyle to german", ...]`` * ``reset``: how connections should be reset when returned to the pool (``False`` or ``None`` to rollback transactions started with ``begin()``, the default value ``True`` always issues a rollback for safety's sake) * ``failures``: an optional exception class or a tuple of exception classes for which the connection failover mechanism shall be applied, if the default (OperationalError, InterfaceError, InternalError) is not adequate for the used database module * ``isfatal``: an optional callable that is passed the exception which has been raised and shall return whether the connection failover mechanism shall be applied (must be passed as a keyword argument, see `Controlling the failover mechanism`_) * ``ping``: an optional flag controlling when connections are checked (``0`` = ``None`` = never, ``1`` = default = whenever fetched from the pool, ``2`` = when a cursor is created, ``4`` = when a query is executed, ``7`` = always, and all other bit combinations of these values) By default, connections are checked with the ``ping()`` method if such a method is available. You can also pass an SQL statement or a callable that shall be used for the check instead, which is then made as with ``ping=1``, or a tuple with the flag and one of the latter (see `Checking connections`_). * The creator function or the connect function of the DB-API 2 compliant database module specified as the creator will receive any additional parameters such as the host, database, user, password etc. You may choose some or all of these parameters in your own creator function, allowing for sophisticated failover and load-balancing mechanisms. For instance, if you are using ``pgdb`` as your DB-API 2 database module and want a pool of at least five connections to your local database ``mydb``:: import pgdb # import used DB-API 2 module from dbutils.pooled_db import PooledDB pool = PooledDB(pgdb, 5, database='mydb') Once you have set up the connection pool you can request database connections from that pool:: db = pool.connection() You can use these connections just as if they were ordinary DB-API 2 connections. Actually what you get is the hardened ``steady_db`` version of the underlying DB-API 2 connection. Please note that the connection may be shared with other threads by default if you set a non-zero ``maxshared`` parameter and the DB-API 2 module allows this. If you want to have a dedicated connection, use:: db = pool.connection(shareable=False) Instead of this, you can also get a dedicated connection as follows:: db = pool.dedicated_connection() If you don't need it anymore, you should immediately return it to the pool with ``db.close()``. You can get another connection in the same way. .. warning:: In a threaded environment, never do the following:: pool.connection().cursor().execute(...) This would release the connection too early for reuse which may be fatal if the connections are not thread-safe. Make sure that the connection object stays alive as long as you are using it, like that:: db = pool.connection() cur = db.cursor() cur.execute(...) res = cur.fetchone() cur.close() # or del cur db.close() # or del db You can also use context managers for simpler code:: with pool.connection() as db: with db.cursor() as cur: cur.execute(...) res = cur.fetchone() .. warning:: Note that you need to explicitly start transactions by calling the ``begin()`` method. This ensures that the connection will not be shared with other threads, that the transparent reopening will be suspended until the end of the transaction, and that the connection will be rolled back before being given back to the connection pool. Advanced Usage ============== Sometimes you may want to prepare connections before they are used by DBUtils, in ways that are not possible by just using the right parameters. For instance, ``pyodbc`` may require to configure connections by calling the ``setencoding()`` method of the connection. You can do this by passing a modified ``connect()`` function to ``PersistentDB`` or ``PooledDB`` as ``creator`` (the first argument), like this:: import pyodbc from dbutils.pooled_db import PooledDB def creator(): con = pyodbc.connect(...) con.setencoding(...) return con creator.dbapi = pyodbc db_pool = PooledDB(creator, mincached=5) Controlling the failover mechanism ---------------------------------- When a database operation fails with one of the exception classes listed in the ``failures`` parameter, DBUtils assumes that the connection has been lost and transparently retries the operation, first with a new cursor, and then with a newly opened connection. This is exactly what you want when the database server has been restarted, but not when the statement itself failed for a reason that has nothing to do with the state of the connection. Unfortunately, many database modules report both kinds of problems with the same exception class. For instance, when MySQL aborts a statement because it exceeded a deliberately requested time limit:: cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...') then ``pymysql`` raises an ``OperationalError``, just as it would when the connection has really been lost. Since the ``failures`` parameter can only specify exception *classes*, and Python cannot exclude subclasses from an ``except`` clause, you cannot express this distinction by narrowing the ``failures`` parameter. In the above example, the statement would be executed up to three times, and the query would take up to 30 instead of 10 seconds. For such cases you can pass an ``isfatal`` callable, which is asked for every error matching ``failures`` whether that error is really fatal for the connection:: import pymysql from dbutils.pooled_db import PooledDB def no_timeout(error): # 3024 = "max execution time exceeded", the connection is still fine return not error.args or error.args[0] != 3024 pool = PooledDB(pymysql, 5, database='mydb', isfatal=no_timeout) When the callable returns a false value, the error is not considered fatal, so it is raised to the application immediately, and the connection is neither replaced nor removed from the pool. Note that ``isfatal`` must be passed as a keyword argument, and that it must tolerate arbitrary exception instances of the classes given as ``failures``. If you only want to suppress the failover mechanism for a particular statement instead of configuring it for the whole pool, you can use the ``no_failover()`` context manager of the cursor:: with pool.connection() as db: with db.cursor() as cur: with cur.no_failover(): cur.execute('SELECT /*+ MAX_EXECUTION_TIME(10000) */ ...') rows = cur.fetchall() Everything else, particularly the ping check and the usage bookkeeping of the underlying steady connection, is left untouched inside that context. As a last resort, you can also reach through to the underlying DB-API 2 objects with the ``dbapi_connection`` attribute of a connection and the ``dbapi_cursor`` attribute of a cursor. Note that operations executed on these objects are not covered by the failover mechanism at all, and that they are not counted towards the ``maxusage`` limit either. .. warning:: Instead of the above, you can also simply start a transaction with the ``begin()`` method, since the failover mechanism is always suspended during transactions. Keep in mind though that this really starts a transaction on the database server, with all its implications. Checking connections -------------------- Connections that are kept in the pool or bound to a thread may become stale when the database server is restarted or drops connections that have been idle for too long. In order to notice this before the connection is used, DBUtils can check it with the ``ping()`` method of the underlying DB-API 2 connection, and the ``ping`` parameter controls when such a check is made. However, ``ping()`` is not part of the DB-API 2 specification, and many widely used database modules do not implement it. When DBUtils notices that the method is unavailable, it stops checking the connections altogether. In this case, you can pass an SQL statement as ``ping`` parameter that shall be executed for checking the connection instead:: import psycopg2 from dbutils.pooled_db import PooledDB pool = PooledDB(psycopg2, 5, database='mydb', ping='select 1') Instead of an SQL statement, you can also pass a callable that is passed the underlying DB-API 2 connection and shall return whether it is still alive:: def is_healthy(con): # psycopg2 knows whether the connection has been closed already return not con.closed pool = PooledDB(psycopg2, 5, database='mydb', ping=is_healthy) Returning ``None`` counts as alive and raising an exception counts as not alive, so a callable that simply carries out some database operation without catching errors can be used as well. When the check reports the connection as not alive, it is transparently replaced with a newly opened connection, just as if ``ping()`` had reported it as broken. Such a check is made whenever a connection is fetched from the pool or requested by a thread, i.e. at the time corresponding to ``ping=1``. If you want it to be made at the other times as well, pass a tuple with the usual integer value and the SQL statement or the callable:: pool = PooledDB(psycopg2, 5, database='mydb', ping=(4, 'select 1')) .. warning:: Every check of this kind requires an extra round trip to the database server, so you should combine it with ``ping=4`` or ``ping=7`` only when you really need it. Also keep in mind that an SQL statement executed for checking a connection can start a transaction on the database server when the connection does not run in autocommit mode. Notes ===== If you are using one of the popular object-relational mappers SQLObject_ or SQLAlchemy_, you won't need DBUtils, since they come with their own connection pools. SQLObject 2 (SQL-API) is actually borrowing some code from DBUtils to split the pooling out into a separate layer. Also note that when you are using a solution like the Apache webserver with mod_python_ or mod_wsgi_, then your Python code will be usually run in the context of the webserver's child processes. So if you are using the ``pooled_db`` module, and several of these child processes are running, you will have as many database connection pools. If these processes are running many threads, this may still be a reasonable approach, but if these processes don't spawn more than one worker thread, as in the case of Apache's "prefork" multi-processing module, this approach does not make sense. If you're running such a configuration, you should resort to a middleware for connection pooling that supports multi-processing, such as pgpool_ or pgbouncer_ for the PostgreSQL database. Future ====== Some ideas for future improvements: * Alternatively to the maximum number of uses of a connection, implement a maximum time to live for connections. * Create modules ``monitor_db`` and ``monitor_pg`` that will run in a separate thread, monitoring the pool of the idle connections and maybe also the shared connections respectively the thread-affine connections. If a disrupted connection is detected, then it will be reestablished automatically by the monitoring thread. This will be useful in a scenario where a database powering a website is restarted during the night. Without the monitoring thread, the users would experience a slight delay in the next morning, because only then, the disrupted database connections will be detected and the pool will be rebuilt. With the monitoring thread, this will already happen during the night, shortly after the disruption. The monitoring thread could also be configured to generally recreate the connection pool every day shortly before the users arrive. * Optionally log usage, bad connections and exceeding of limits. Bug reports and feedback ======================== You can transmit bug reports, patches and feedback by creating issues_ or `pull requests`_ on the GitHub project page for DBUtils. .. _Issues: https://github.com/WebwareForPython/DBUtils/issues .. _Pull Requests: https://github.com/WebwareForPython/DBUtils/pulls Links ===== Some links to related and alternative software: * DBUtils_ * Python_ * `Webware for Python`_ framework * Python `DB-API 2`_ * PostgreSQL_ database * PyGreSQL_ Python adapter for PostgreSQL * pgpool_ middleware for PostgreSQL connection pooling * pgbouncer_ lightweight PostgreSQL connection pooling * SQLObject_ object-relational mapper * SQLAlchemy_ object-relational mapper .. _DBUtils: https://github.com/WebwareForPython/DBUtils .. _Python: https://www.python.org .. _Webware for Python: https://webwareforpython.github.io/w4py/ .. _Webware for Python mailing list: https://lists.sourceforge.net/lists/listinfo/webware-discuss .. _DB-API 2: https://www.python.org/dev/peps/pep-0249/ .. _The Python DB-API: http://www.linuxjournal.com/article/2605 .. _PostgresQL: https://www.postgresql.org/ .. _PyGreSQL: https://www.pygresql.org/ .. _SQLObject: http://www.sqlobject.org/ .. _SQLAlchemy: https://www.sqlalchemy.org .. _Apache: https://httpd.apache.org/ .. _mod_python: http://modpython.org/ .. _mod_wsgi: https://github.com/GrahamDumpleton/mod_wsgi .. _pgpool: https://www.pgpool.net/ .. _pgbouncer: https://pgbouncer.github.io/ Credits ======= :Author: `Christoph Zwerschke`_ :Contributions: DBUtils uses code, input and suggestions made by Ian Bicking, Chuck Esterbrook (Webware for Python), Dan Green (DBTools), Jay Love, Michael Palmer, Tom Schwaller, Geoffrey Talvola, Warren Smith (DbConnectionPool), Ezio Vernacotola, Jehiah Czebotar, Matthew Harriger, Gregory Piñero and Josef van Eenbergen. .. _Christoph Zwerschke: https://github.com/Cito Copyright and License ===================== Copyright © 2005-2025 by Christoph Zwerschke. All Rights Reserved. DBUtils is free and open source software, licensed under the `MIT license`__. __ https://opensource.org/licenses/MIT WebwareForPython-DBUtils-8de754c/docs/make.py000077500000000000000000000017621524214146500211440ustar00rootroot00000000000000#!/usr/bin/env python3.12 """Build HTML from reST files.""" from pathlib import Path from docutils.core import publish_file print("Creating the documentation...") for rst_file in Path().glob('*.rst'): rst_path = Path(rst_file) name = Path(rst_file).stem lang = Path(name).suffix if lang.startswith('.'): lang = lang[1:] if lang == 'zh': lang = 'zh_cn' else: lang = 'en' html_path = Path(name + '.html') print(name, lang) with rst_path.open(encoding='utf-8') as source, \ html_path.open('w', encoding='utf-8') as destination: output = publish_file( writer_name='html5', source=source, destination=destination, enable_exit_status=True, settings_overrides={ "stylesheet_path": 'doc.css', "embed_stylesheet": False, "toc_backlinks": False, "language_code": lang, "exit_status_level": 2}) print("Done.") WebwareForPython-DBUtils-8de754c/docs/persistent.png000066400000000000000000000155761524214146500225700ustar00rootroot00000000000000‰PNG  IHDRšê úxšPLTE@€ Àÿ@@@@€@ @À@ÿ€€@€€€ €À€ÿ  @ €   À ÿÀÀ@À€À ÀÀÀÿÿÿ@ÿ€ÿ ÿÀÿÿ@@@@€@ @À@ÿ@@@@@@@€@@ @@À@@ÿ@€@€@@€€@€ @€À@€ÿ@ @ @@ €@  @ À@ ÿ@À@À@@À€@À @ÀÀ@Àÿ@ÿ@ÿ@@ÿ€@ÿ @ÿÀ@ÿÿ€€@€€€ €À€ÿ€@€@@€@€€@ €@À€@ÿ€€€€@€€€€€ €€À€€ÿ€ € @€ €€  € À€ ÿ€À€À@€À€€À €ÀÀ€Àÿ€ÿ€ÿ@€ÿ€€ÿ €ÿÀ€ÿÿ  @ €   À ÿ @ @@ @€ @  @À @ÿ € €@ €€ €  €À €ÿ    @  €     À  ÿ À À@ À€ À  ÀÀ Àÿ ÿ ÿ@ ÿ€ ÿ  ÿÀ ÿÿÀÀ@À€À ÀÀÀÿÀ@À@@À@€À@ À@ÀÀ@ÿÀ€À€@À€€À€ À€ÀÀ€ÿÀ À @À €À  À ÀÀ ÿÀÀÀÀ@ÀÀ€ÀÀ ÀÀÀÀÀÿÀÿÀÿ@Àÿ€Àÿ ÀÿÀÀÿÿÿÿ@ÿ€ÿ ÿÀÿÿÿ@ÿ@@ÿ@€ÿ@ ÿ@Àÿ@ÿÿ€ÿ€@ÿ€€ÿ€ ÿ€Àÿ€ÿÿ ÿ @ÿ €ÿ  ÿ Àÿ ÿÿÀÿÀ@ÿÀ€ÿÀ ÿÀÀÿÀÿÿÿÿÿ@ÿÿ€ÿÿ ÿÿÀÿÿÿ 000PPP```pppªʲbKGDˆH pHYs  šœ}IDATxÚí1“㶒ÇA%–^à§A)ðÎ&{J¼Š&Ól õUÙÙ»äªø´/÷‹îj‚;f¾dm]â‹žù! å„Db—C7ŠÒHiD‚ õïªQ$Å&ðC£A²Ùâ©TÐ@€h @4 M/ЈO²ƒmÔ¿ž h€4@ ÍÉ QE8Hìš)/哤òí°“Çê·‡øèž›IrðH›I4G$ßÍÕGúM`ë(§ZäJÎÕº<8QwùCtôÛ^ ˜Æg­šjýüDhž>L·Vo¦¥Åœ¶š;Ùµ¡=ˆ¬#÷Ìb¼³šÉ?T«ÿÔ¶¢þòà·Pq6ý¯…¢|ò«s™Ý}7ˆ µ^Õf*„)pB|£ì$_ˆA\~#ÑÛ‹/? 1–òó?èWÙ`(Ddö(´¥S-DjIÐT¬æ·/‘|+ë VœO»¤½CþE>’Q¡Ö¤A\¬îåz” þÆÆ£·Ëå )Ây± ÔÆ™>.ýF—öȈ:ëãýIƒjA4[« ’õ\>ÍÕ§µšŠý¨º1í“M"ö ÆŸ“8%ôö)BûmÛ·)fÊÒ‘TÌäÓX—ö²{„ñVío5ÍÖj’üî·O;¶Â¾f3M¬¯±ÛTo$‚Gþ ¢¯Cö5¯ŒÒaRú½½P6&³Ño+úêãÒox\Æ{X}Š—âhtMÕç{ŸTl¥º$·¼l›Î'qÅNŒ/¯XÙn­†­w­†÷˜ìXMiC@Sµù$fôIþâk íç‘ypÃVÃת–Õ§ò òëkÉ×/b|Ù.—#Z[¬Ø;éëíkâLïA„6VK¾Æ´ ©ø5ºbB§Ÿ¨+Q ^ˆ¹¹¶áöÌc*EHˆŠ’ê·¾Õ#4ÅAù$ñÁø½½XýÁ#®ÅŸ4P³6¨Gh µÇ"²Zì VãHØרU„5¾¡Yé‹”%Ðx'Öjî4@Ð ¤.4_kûÕÜP¥¥Aa€¥ñ¦ ¢Wtúƒ†¹ ñ“Íör hüBÓ/2@4@sËãM  y \r Ð\Êh€h€¦/h$Р0ÎJs4¨ëÛX¿ƒFB¼3~ ñ h ·:¨Ð ÄãKÎ7?œ¼°Ð’:çújEãÖj«s®¯ÎKN   švÐp.?!‚ÿœœL%k“öœÈ8{šÌ¨û÷sò=›œZ× ¹H_&D{e5œ{ùdî°bb6æ‹ ¹ÖjXÓ9ùž)åV»ÒjÎÖGyAgRm [Âfz†Õ¬Å»éÕh8·ß9ùžS΢]‹æl}JÞX>÷Ÿ©÷Èß-ƒd›k™2+§B|k¬æ÷äDÎÄˬæµ|϶Ö¨«9S_:”ÞYMŽÉš³ð^Ú\ËÔŸ1åM^uùš²%¼’ïÙî¼%õXÍYú²ÐK_£ó"›<Ì”óÕØw6¬øšS}öeVóJ¾gKfpvUÕ¢Ï[_cs1ë\ËüO´çn>‹ŸâŒ¯BcµM^Ѷóµû•hò‘¾Óñò®O~WýÌF2ÿËt~Iõå º¯Pý_ë%ç:*kå™Z*=MûyF­M³joãb%ø©ÉçßèTñ÷ä4'µ=W´éö~TÛÙhþ'.d<—×Ýwšqw=Wç¡ÖüE.…y~VPù>'T_± »‹E¸ý_'>Ÿ?÷£:[EüLC-}Ž³÷¿ê5Öò?źrߎ樶Šmzߧ+ï¡ñ9?ñ½±| iÆÐ ³*¾Dù€˜©5Ó¸Ó;f6-ܨ±½ÕbS—,fÜçyT]ÉæN¯)}ΧD·å·£ÙöIåØë˜ú¹»_·ú¥sù:»9÷|1H艳øÈg°Ž6ʽg…óíÝfjkÐXkW甎ËÚËFOÊ]¿ÿ:Þñ—ÚjæW 9_›ñFëùuhÊzOgÊ›Ø3HÇ+åòï*kô>ã³[ž 4d ôì¢XEù„æÅä3-~x§|õÝÿÑ ¿Ñ¶'òxs^‡¦´e¬í>§cïh«è/ŸÙ=Ý_‡†Î™Ÿ*/©ï,–º¿,Þý@Ïf~U½©Y#Ÿ–_"ýç ù%¦(&Õ“°Ÿ™v¶s‹ã5ùbX±šÏW V±~3>¡úþ÷5R>\; X›©Õçü166ò4bßÃkŠÇ+Þ§ø{ìšüýEÚ²3{˜c}ÿì"méÇkÏÙe ³™ôÍÑÞõ0Ès¯ÉŽ©Kǵh;ÿ’ói~ÂâNú„7j»Q4@4@sshð´ó7 ‘ÙÉ[ \…4†Gw YÉ0=+Ä©Õ”ï™Gƨ1ÇhøùŽÜQÉa5í ÙÉËp2酪KÎm&€É½Ì@o”fCTŸ[4ÚjL&Š»RßÍ»½ôNó€}M¾˜£îZ°š½÷±«™âµy[õrt\[M°— x‘ @ÛP:€³iÃ×PþŒÝLÉÀ¾†zºìT.*Hc¾fk5•L5™BŒÐZAÐ h€âšZ§3Ó:^µ^6f/¢…&ÖC@4@4@4@4@4@4@4@4@ÓS4Bÿ gg/œâqY¼ÚÑ¡Ž)\¡ÑÊ¢qX<ÑÀÉ áCe®56„ÆmM9G#h€æ&ÐH÷hÜ;ŽFöï´utÆõÍ  ¹)4nkÊ9ÙY4¸ó\§ž‡£ŠŽŸÿ¡õ;h$Ä M?  Ð@€h @4ž ž›ž’¢Mmíh¬M#»ú¡­/ÑH     ñ Msƒg;G 44›Hä åˆ}ì(š"49¡ 1ö¢©dVîšT|˜:³šçQÂó ¸íÐÌœ>^¢YüCˆ‘jA‚{“,ä\÷ÓŸ8½ýÿ–óáÔ…æ³ÑîhûÑ&Ó'Ë©Í©òQëxÛ¡«A’/fT'ʯ‹¹Ìñ&¤¤~_S¬¥cV,"¹™*½‘ªš$?Ú~…f>¨Íéòe¡Ï¾f©ÚÒzœê:¡yŠIlæó²v«Yìh›ìi[×çÙÎ,ŸÏ¾†fÙðë°Úß›ù<š°šª¶ŒµÝÓ<¬gDµž_)Ÿß¾FÚÓ~;Ö34`5/ÛpXj[—3„ÔêkN–ÏÎç§Õ ¥êóu_ühûbšÏcÚˆÕ ©¿g_3‰­¯1sîL¢.9O•Oé–Ù{ÏË?…˜½35f5òÄm‡´=ñ#­ÚGh§Ê·¢^ïö’е¾ÆÙsÝâðF»òÕŽ†ûbw÷Ж®Ñ¸+_V9µš¹s«é,šúwõC[;h€¦ÝÁ3Ð @_r Do6½ ñI€h @4§hh @46ÑlLüå’?Dg>ëz)Ùùj€æ@íŧדøÌg]{H'ÀršüEÃÞ«ÓâV‹9ÆÄö¡™¿[I*lô/©-"¶Û'? 1£åÒlÔ>߆Êjþ°ÑÀ}B‹9ÿ.~¦XŸœ/8…ãTÔrQLY.èxúØü†„ˆ¦Œí¥VŸ“,œ³ d£„b‹ÕR:âx¯rûÌÆcYZD=¹%&&œc¶¦1ÅŒeþÅ6>™ú½µÚSÇ.“Õ¨?ý†=vñËl4ÒÆE’(ô戩aªÝrIV¶«v^ñ5T‹Öׯ}M¾ K/gÞàþÞΧQÎÚU[‡F3tD/æù03Q¤ÁcýƒçÓåóÚ×薳ߎí|yý#4Ò6|¡Mé±6Sû0àDùxö>Oј93¸÷ŸDå|[«©·CSm”æ2ÚUmùvfÑœ*¹›Í¿×5/ÆL¦nÄ×ì—Dež?Òjb„v´|4IIR/éìnnÔ Ð Ð Ð Ð´rÉ 4@skhÞ~Do6½ ñI€h @4§hh @4¶ÑÛgî••‘\¾qÂt/¹‰LÚ3ò‚W±š_®ÙÄ @.Ds¸Þ(~ñÕo±šÃô!ÇÑPhHTp`”“ ý½ØaW 9„†*-Hʸnjݱmëd5ù§ÿþR:}²‘ÒjfÖŽ*V”–AV£ÇU« ’—±Ã&ⲇ†ûžŽ^²õÐû*¿”¾FÊ_þín;:SžÅøš¼âkLDð`ƒÌõN¾Æø!u|ÝË%»±Ã¹Ž+CVÃ#4åéU¯¦Çir)ÄG¢ŠoÉód¢2RV}Ô÷“ƒ#´j ré¹ü.ªÚFÇXŠ¹í7·±Ã&®r¸CÃõHGѤ#T—÷и¿¢‘3µÊ7å!^ ´ƒ4 éšžMÓéõÍYMŸg}ÝV4@4@4@4@4>©úÏÝäâÂa}¹UW;žÅÔÙ¼ïZ™34NÕ‰úO¿ÖWN=RæZ]ChÜV–ì¥:  i0蔽T×  47Ʊ6Ùm4Ò5)û©h€h¼@㶲dOÕáγ·êDåRª“á(¢ãçhý ñG€h @ÓOB@4 Ð ¤ƒg·ùöoc>ºÐ4²«ÚÚÑx    ÏÐ\7x.&Çv±9ÔjDs0ç4ë …I¦W3š£åKËW¾¢9˜;]oYÔ7«økÚ(çmªkªf4Ç4R¦J“/Ô_«9ÒŽ×âÝ´v4ÇÚ°É•ÛˆÕOŸhlØK4:wú^nô€s£Ëß›¿.4{Úvò«Ë©Í©òÉ2‡²ht>ÛÇBÕ‰ò+œ}ÀÏSN?[³¯¡ã â|íåWÏt¿òuX·¯9]¾M豯1Y æF§f\¯¯±Ú†/´±ž´>mg–Ïg_“êÜéent©s£'f†š¬^_“ê|ùG´¥6qhN—Ïú!Ÿ­æentnÁ5ûš#mXçWOË°š#å³£7ß}Í$²}qÅjó5&{™k³G5äköË—¡;o‡6wúv³Í^¿ÕÐñ?¼Ð¦ç/Xñ#­ÚGh§ÊG—œµz·„ð(ͧ5@4@4@4@s³hì4@sš7Ñ›ÍGoB| Ð Ä)Ð Ä=OíŒb«vç«Í8N%·kmŒŒy"{±ØßCÎFÃOñóÅØDZ˜ø*%Ë?GÒÆ_¤AdžöW"ŒÏ)¹Ìj¸5?L,lç›M¥-¦Vu –,#²`1.¬&0DŒ}ŒÍ~éH®æÖjŠÕÅàk«11X–ÚP0ŠÎÃ#¦ŽGÛ´m3µñ[»±Êko"ð Ç} Ó»#Úh¶=J⊿°Ñ’ãé÷j í]„qªãRMü–ÝSðžÖö  m­æÁøyjçlElBc™[c)­& Ì{.eѳÊê“c@ã­BѧÚr(º\Ñg2 «lx &úL¦Û [ô™L·ýè3™n£DŸÉtûêCô™L·/ÚEŸÉ8*@»º±îh¼Õ#úL¦Û—¢Ïdº}A-úL¦Û·¡DŸÉtûæ­è3Çïúh¢¯d:üÀ[™ hð±Cû±“Ý}¬ÊhDScr¢Z[z®*jaSw>ÒŠG¼I4ºQ ¿Ñø ÞÊ“Uá3¡Øm¢‘õ8‰ÆŒFøñúD[Oðj(cFÓTOÙ_SWáEs«¾¦64MÛ ûšZ* 94·ìkj©‚êÐ0­>_­ƒMo‡g-_õ^Æ»3ê šsš½hNœ˜agÑØ»6n¦ì9ŸzïÑ´1cRÎ\4X.Ù{iRÓŠÉ[”*@G“˜œ£5%ÜZÚrõ+qÔ7uü’¹#½š;«¯¹Ä×8´ŒÐ.¹ºFƒôÙæ6Ñàn€À·Xv?P½µDÞ¡y­`Ü×oT¡÷ƒg[±¯fzƒhpÉ 4@4@4     šöDÏè:;¸mM‹ÂsPç‹ÙÎw-4«1дh5<uVÎç]Îä«iÝjxê‚æôVÛXÏ1ªÅ‘²š?x2jýÍÎ-­öh}^ö[²šbéù×cú^,îÕbT,GI±ˆèÛ³ú6y”ÙPíy/×£hùÛwå‘õ5E³¯yImUùbn÷WۀƙÕ8A0Aw#†.€€;wîÜ!¸CPï¸3š¶ŽpÇ؇¦îÚ;Ü!¸Cp‡ àAÀ‚€ûhZ†/^bËô“=+ùn¿ô3[ùŠŸ½ìÖÒ+,-ýÜ_Š–>ŽpïLÑò-^‚ïRî"Cé%,–ò=»…['£jåUÚ÷V½ûê§âeû§”=ᅣij{åLÆË—B_¬}Žk߯àÙ{×Þý¯Äí_—¾¦;Zþj1à Wÿá0Öß³yøðÎôb±_0Ææb00öðç‘ÃL/Ý"©ÏãŸblÁùç¥-ù»â\a1æêâט'ë?ëÒ¸£Š àÞªwÿõg—¿.´OW1¶|·’Ñvh-ø«éG– tíò`Æc牿nìˆ-Ë7.ßÍýØ’[ÒÉ«ÏùÚ{íx#^œGõ»TJü®.¡ë“[TƒTü=po×»û[›oíèà݃Ÿ'=9–®Œkt|[^0÷i‹Êˆ9ªÞJãKŒ᪃9¾[¨ß¥R‡yí¡>*ŸÔ÷6½»·ˆ%ëKïW~ÖÏ{Iü.?чˆrä«á3âü%”;ݯ³Cì®>—œó_‰úpöù»ô9Æd ]qŽ¸8/†€ûíÞÝð£ç¿ïùô¬w7ŽÈ—ç/ãÏ#d¼»þ<ñîNêÝ—‰w—%,/SËÁ×ãˆ÷V½»Ï¿°G®#ëÀPÞÝ£ëî*¾ðÓk5Â_‹Wgó­ˆÝ-»?¾+aM>_‹ˆÞqã Eû®ŽÝûû¥G%¾Pìî%±{hú{5_ÀîíÆîb†(¹£ &o?Q,o˜KžvÍìL\ÃåµÎÆ> n#Æ>¨+3"r¡øä­ŽÝÕçñæ7y¥eý;“£IÆèâ;tef-J¬Ý¤–äÊ ¼;p¯ÖjR»ÁÅEà>}%œ¯]ôp¿ïÜ;wîÜ!àÜ!àAÆ‚î'iÔ½³Ðçèz}Ž®‡Ðçèz}Ž®‡Ðçèzô9ÔK·ãºp¿—n—¬3ô>p¿ÞÕ> à~¸ƒvàÜ!àÜ!à>zÞÑ À¸CÀ}‚¸£€;p‡€ûyG'wàwà÷1ãŽ>îèz¨‹>Ï[¸ýØ?©ýÈ3ݧwGg@À‚€;wî4Ö;Ü!èîpï qoµõQ!4lÜ;›8t_[BÃÁwàÜ;pîÀ¸wàܧ…{hÉËÆ?®üâ¯ÆÏž~-ý[p¯VÛRÕ‰Âî¸GºÂ_–6OUó* ºÀ}âÞ=\y¥‹5<¶LïFï®j3.×[.™w³w—uÉZ/Œ¯péòÀô€ûøp¯s!r¿ºÌË| ؇•#îr\…Î%ÁÜçñƽwY×~UÁ»Ç€ûDq×Þýc6Þ™^l1òˆˆæâcƾÓ<üWÙ¨`Mצ=üÍÞ]¯_(2Š¾_~ä0òá¢&¶õ¾µÒFíæ>pŸ6îŠkÁ_M?"¾Öâ¤>ã±óÄ_ 7vÄ–ÑPìN¿ Ï%ÖžÔæÊÚ¬lmuè+÷î‘h^hø¡õ$ÎOô»Tahxô²K"öÈBì~/ÞÝ#–)~¥í4~ö(¦Hc÷Ƽ»ˆÉEL}ZÛò¤¶]Õ™ÂåØ]Õ¤ë¥véYB8;Ì)‚Bì~7±»øGï÷t5CxXz5Ü`槱{i \7vÿè½¼vb¼ÈWÃý:ËÄÏ»b_{Mì.þÉze…®|5^èüØýþ¼»ðyäÝõŠKïÜß6»ŸÕÆÎj+ñíWÄî²&]or†¼»yäÝ-î“ŽÝþ–¢iñ*"y¾=¦oÝ÷ÙÚÄKLµ­dm_Žj#›À]ÅîÉøòeì.^E$/šçÑ EŸEhOô€©ê}Äî©¿‘-£ëÝcÅD.bìCãÞÈ2ÒڈﵨȷØ[UÛVþ¨+3FÆ»ÓûÈ¡X&Þ0ö‡M1͵w3|ÜGDü¸7^y÷µqÜDÜ;pîÀ¸wàÜ;pîÀ¸WVäЕ§äæÓ†Žp"î±¼'û?ÜÌ@à¾û(ï…=w:ûí~ЯŸüôýõ¸Õ…ïÕŸx¢é•þ ´Ðïo½b}ôW-·¤>à> ÜÃG}¿ŒWă~U+™Dñ›pÞ«¿pžß!=d_ƒ¹ Þæ}#îUë ç<úä×܇Œ{õãµsS®_¥C}X¿<’WÐágå]ç?üJw»ÄöoÂý?½”3átßððamþ$ÛÙ ¹gÍf‰ibÿgÿ6ÜkÔSû>ãž™)ã.ñFÞ+BÔo\yg¸x÷Ù ßS{’³þ'O p—¿±•«‰"S¼³å}ö<þÙLâRì1³e‹ÜmÕ¥ÙÕë dÀSýt÷$RqLB\ö(ï üùE-ûµ'á?ù*¾wý[‘cú´Z}”õïܽˆ°ß£=v:ºäb»Ûp¯QŸ@ß-®¸O÷Ô—ní`‘òη¶x÷uq4§“Þ}kß‚{ÊV°ÑrR°ØˆiâƒÞ£ÕѶ2êjܫקGrQ}À}ÁŒpÙt¯y¼yçyñ¿ô‡ñßÓ½ßjOƒŒ{øöéÜé7è¾r¾–QÓZEJñ?øäm{Ò\_Ü›p¯\ßëBÕõË;&=UýY¸ìÅ,r.7×þp³žQYÆ»¾mªJ¡?…¶|YhߺK¿+÷ÄÍŒåÝîbfìÝ6U­ZŸ˜¿P™Âú€ûq¯~!2z_«–°â•º":Š/dÖ+^÷¦êîÓÀ½xî™;8~¼uñ^á\07yà7âÞT}À}"¸7XoµqÜDÜ;pîÀ¸wàܧ„;ž¼‡'ïÝî49â;Ü!¸Cp‡ àAÀ‚za=‹û¾ÎE!hܸóÌs+Šq·Ó šî5ž A£÷îÆÅ'Š.=¹w®¼{ö‰¢núDQK?QT>•ˆž5ÎÐ}ÐØqW±»~¢¨znjò”¹ä™s–96úš„w?y¦cö‰¢ÞN?ÓQD8 ÄCñî‡gΚ㘱æIo ¹Õ[ M:˜©ø`ÅŠ‚µBzv$±6[ ÷ª×¹;ñ:öØõ³`àÜ+á^õvrÖ*釸÷¶p¯:…¬èþo&îê¸÷Yu¿®ãiÖé·ýhî¹ – Vœ›^üY£'"àÜÃ]¯Ö®ðãï¶ËØ­¼ƒà~„¶º‘U_Í âü–¼éõ¼ƒvà~ B–rR&t7wàÞ îµb…£\ G¯­·ûªJ€;p?á<™M%س]µü:Þ pO~(7yÓÚk’ﯻ žqo 5ËWQyÖT½Øý•S;Çq1Ü„uåÝÑp‡ àAÀ‚€; uàwîÜ!¸Cp‡ àAÀ‚ÚÀ=·Àˆnöœ¶íéb÷ïN_ƒû¨†îtl‡ÚèSàܧjÿ+coÒ8DmÂÕOŒ}äüÄ»SÜËÊŸ´¾eÜ㟒'ÒP™²¦x¢GaLѲ›{÷òòŸÿ²ìÔ»—õãgï÷’ò‘¶¼Ücªy!ûë¤fûR/?I{©—W²—ãË_3å»Ç=Þ~ä<¥½³E àE–­ÏQg±û…òt^íÌ»;¦[6Ù ân²HØZdõóYì^V~kw»—õcôüŽ1ï÷²òÁü_šš3åá.k^Èš—iÍLq}6Ïôò:·—²—ãž“4`èu¸¯E?í²Bžm–/ùª£æÝ/”ߙʹ¦’w¾‚]„3?™iÆÏ.Å°{eôiì^V>úÑ:õîeýH6ÅŸŽ§ªeåCÉ^k±»ª9T5SßÆÏžªÙçI_g{Ù¦°6œéù¿O±V´|!›cçñÊؽ1ÜɺÐü:ËFßäÓÕÑ–y­),¿ÕµïNÇAx6udÿ:.Q¬ýôYì^R~kó®½{i¿'ñxÆ»—opΔƒ»ªyvRóR[º[œ^ˆ<ïåuÒËdoÜÐœúøG*Çîjì¾f¼†»bL*¿~»—•O|{WÞ]ĆÁ›Œ‰É¤ÄÄÏîÉH-,ÿ«%¯¨=u»—÷{–Š¬Í-/Û»vÛõîÙš—K“Yõñ94·—W¾Þê÷|/3ã"Î’‘ó’È,e&'v/.Og¸¶pÏ»éÌ}a‡Œ—n%*?BqedžÆîÅå;¿î^Öå†ïO‚™²ã´Õó•–bw3­y™ÖLýKl8î™w/éåµø²á÷‰ûúwyeâdÞíÇîÅå·ÒEº]áo~“×1è€\ X’þ}eì|¤–”?‰Û:ˆÝKú}ÇXf¤c÷’òbØÞT•¯“5Ÿ\ò“¸œçô²}týKÆíêŸìó^½»]ù‡ë–oÛ»¯ÝZ¦×)ßMì^í·tÆóüؽF/;mô2k÷N˜y•nAý÷Ô wª9RŸjW-í•–¿V´¼Nì^÷kúý`wÈÜkŒ?\j·—/[ÚîxwÙ†@žâ*÷{•Ø½æHu®$æ6Ëûóî»f_{Mï~?¼h)ÖªŽÆt¬fj%˜îÀý^ïwîÀ¸wàÜ;pŸ~ì>ÜT\/D8‹ØåÅ{Î"ƆÙËωüîC1~°fwØ>U5±VÈÀvàd&wàܧ»wØÜ l¿ÜÑëÀ}¢±;îÀ¸'r¼’ƒÇáâ~l[°à›Óû«ã÷G›;{¸k;…ý¥}UïæʸGŸüâ«­_é÷Ûò,Ý|<úÂf ¸§ö—ö=N«­àþPöásû¸ë•ö9’³j¶åYz·Î`1Ü)ÇÐC—Ö×vò‰ê·ÞqϵêbOšýEf‚~]|‘k¿È5ïbËp·2!íù‡¥ßßQyJ Ì׶iK·rëuA«eŽ˜-e UùOTr¡Š|XùƒÀìTö‡êÝ¢o°N=N¹ŸéÕª«p§}”*0½€¹÷R¨Í‚ÓÖv&H{äÊ2oŠðÁbØ0›Ïé·¹câ“wbÚ/Þ›^ÈÔöì[„Œ¶mk KÉ–ˆ…·”—eésJQ)3QPMk›Ú9³aLUƒ´oƒEb_«þäû5û¸f‹è!úÎÑ9J)&KßGÆšÞrÏ–}ײAœêûèˆc/mм)«v«¾ÏXµcÆÚ¾÷Ø‘ùwÈ…Š÷ÿ¸Ìµµ¹Ì.df¤mcŽ'’y ÷у¢¼£ÓpÎÅ{QŸˆaÕ~™§Qxû-e› -»måCUmÍéKzÉõÿª¸]ü¾ü,œË6mÃÀ}ë&}»qµ}ízÉ%å¿ ß{ÿmåÇKŸ²wíÂwÞ/]ñž¼©°måGVË!¬¨x‘ ‚;ÑQj•€!µê“´jC¹ööÖMÞÆ8ãÿìrú·‘y"A¢zƒÊdÙÜäNX™òKæ´] ¯&ÆðB³>{2WGïeý6]EÙywm[b©)·D+’|H2«“MÖ‡ê¡2è@pvjûã?{aúЋ}©ê÷‡Èq#õ’©¾ˆeU$©öK[éz׶mïþð 33Q^´$¦ÍZeæYueâ ™ÇuûD¿,Fåý‹ÕÜ)|£ê]Ð à©á>„¦ô(b¤Ñ©E0î’_­’ûõËö= óªlÓ–ª-W·Bø ¢)~ö(ÏÉ«"‰JÆ–; ÜÕ,CØ/Îc¯ AÙï\zwéYTLL¾LÞËýÊò3_ZŽÝ‹þ›º8ŸSFreÉ+Yõ’µ*<¶êZÜ#¥ å_‰­áË-zº‹aŒÂ;¯é~]PFf1Ë>YÎlj•ãr?½§úe>Þ>¥íÒ¶ÄR©lÅwåó d–sqþ3=ºCŸÑ£÷‹ªòX;µý¢ÍÚ¾N|)ùwò—Ô/áLDP¢ïÔ{µ_Ø¢r SVÇVqOÎ7Ê1 h.#,1µUò½>1OåVV]½¼ƒÎ¥ enÁ+ïë£ú€ò~môZ†ðᢅû‡Øbo›…Š]é9[{Ù*¹ß¦¸êWs“w…yu´m"š¡ù‡xù“²=Tù ׌ž­;ÆKÒ1ú‡•?ÜÉNeð˜vvËÞ}Ÿñî/tN5=ì-å6¥÷¡Ú¯:þ«èø.¼»¨õ¯dÃ?­ ”CR3«ÞS/¢ÜäÊ…²jê«™`;t¡ ;lî@¶w ÛÇc+2€àÜa;p×û2雊Ò: eÿ´lÏý.ÉþAôíÜ!hò³WàwîÜ!h*¸'φV:_ùyíªìãß… Nq/?:Z‡J÷g¶–^ŧb–ý& ÷KÞݹ»óCm)úÑ×ë°.áÿü§dÅ+­qPϣ߯|;òþð€±·´îS3+¶h-Ðot?¼ÚcɇÛ'ßûE®¨^ý$?yÏ9­½sÅ{åߥû•_„¯—%å]èÐmÚ?TöîkÓ‹-›Öƒ¦\%>ybŸ´FßæÛt­míh ­0q“=zu}Ïz¤_‰,›¶(ûÇN•¤ÕÔâ7ÏJÊ3&Ž¤|çû“œk›Gr]ŽL:~°('…ü_¯í ?ë¯Ù,z eÈ¢ô÷2£Âš¥Ö>öî‡o"òçK/\©u©D§|'×v¦9‡ô–üÎöM²GF*ú{ž${éÒŠÖEÜÿª2tÐÖáwuÉØqq¡ÃüŽÖRF´6ï“Ï¿Ø_’>ɧ⒒¨ÿ)+ÅÍ+ŧrEÖÊW«¿q2W<öî\0ÿJë¯7¶¤wM«A#y‚'¼s&vßê-'M¼±S,¾G<£1"Î(;S0KcwñOúr')Iñ³qœ!ÅÈIN ‡rRȵشRz㪵Ëj]¶ÊJ‘®ÔŽh-+ßÚzm­ôÿ¹Þ] ûu!=ñò…£HÄØáJs,÷g½;Åî®Ì<£?×,«ïqý]íݹÎkÀyæ³HŽÓäZMh"x‡’\®UýfÊ|22'…ŠËËjí²ÎO!Ý­xŸ|ïÛBæéQkkiNéæyw‡çÆîz÷./v2±ûÒÍ|¼»b™bw­G*7@&v×%™Ì€ %Þý4'qÌ¿­< Deq$OkW*så,}•[SŒ™+î1×»ÿ&à }ef/¯°H_/ßÑþ?®»ÓÖÒK¾“\™9úžŽÏC}qÇÕWfhö |ºº†C郞Ì@‡Ø]椈('ÅwÎßÊœod–<ŸòE¸\ùuE4åüÔ9ô(Ë  > ¼Ÿ:—˜¶ÎxQìAÓË‹Ý!è>p¯tÿ‹¼Æ"s`CШq‡ àAÀ‚€;÷+0>ü¤îgc¶ŸM÷¸TÛ«Og}§å1?NãÙ(z~XÝ Ü;pîÀ¸wàÜ;pîÀ¸wàÜ;pîÀ¸wàÜ{§ð%S¯û%6 j¯cÏ€Œ¯gø5_hÊÌauï ¸Kó«û%Æë~©ÅÖ3^Þ_Ïðk¾Ð”™Ãê^v[[Ø5¸³á„µì’ñµméÅøÁuïM¸×¦ýÊ/µy<êØ3 ãY][X/Æ­{;pîÀ¸÷ó:øNëÇcŒÆ׶¥ã‡Ö½À¸÷Êæuó¥¡PÜ[=ÜÀ¸÷àίÄóañ>NãkÛÒ‹ñë^àÜ;pîÀý̼®¾4j€{«‡{ظãàÑÞ›ñC½øŽ³Ka{ûÙ ²Ò!3$tGîÐ]âð!àAÀ‚€; ™uàwîÜ!hZ¸³º¦|ûm{ .q¯õ«õËwÑÖq7îÀîÀî£mÀ8&C·¸÷Q?t+;pîÀ¸÷KŠ–~¦D" s‡„{À˜áÖ:áÊî÷Š{ì˜^õ¦Ä–ËCãe8¸‡K—¦_çXl ¸÷›ã£†wæ>7ö°‚™xéÕ8!ƒwŸ4î¯ê|¿·dZ´®~bL`û?âÝ)îe嵇ï÷PäIK˜²KlEË_{”evóï^Zþó_–ðîÆ=zvÉ+¥"N[œý½H°J¿~»_(DW'ÄÄ–ÇC““%"îVñ”Zd—0×:ÝKËoíÅî‘ó:‘Ø:ñ9âÈ+è‹Zó;Ãë wíCÕ¯{áÌãgb^å׎~…ò;³+ZÒñç(ƒf~2ÓŒŸÝ½ˆ`öêÄt»—•~ô£áx÷pQÏonçþ>¹uçݵ§«<¿ ŒF]b­`&–'õ`–¾é˜«ã~~e¦´üNûÒ.ƒ™È!§¦Ï*!Ùå¸D±©g±{Iù­=¤+3Ûz¸Óy·—`F{ÀjVRÿjK{Á]ÎWuô‰YÁyêÄ»çLU Ëï”'í|ªújúáL÷&nÜÃx¥­³”ÿÕ’gæ§q×3¡(o&ôÙ;ǽ¤|â™ZÀ½ÌÊ4~­f%_»\÷}ó¸_¾©®Õy*÷’X\œm¢üؽ¬<}Ö9îóhüÉX|é&±¸òÖt–Ì“ÉGIyÞõu÷ìLÈHgB–œ EÏïóŠgN§åƒù¿´ðWËó5}-®òüÎa3ÞîÉ_bNF¢_»—•ßJçØÇ•™u¥Å–v*n§_Åg§®§¬üÉønž­&Aö\ù¢fBÅáñ§ã¨¸¬|(yj#v/Ÿ¯é9RÅù]l}l/vÇMÃm@ùLHFb%åÛù«G…ùÚ⬋KÊ·»÷âáüþ«ŠŒûÑLHûÁhu2s"ß7sÊ-/Ù[»màž3_{I­Ü>æuqQùýRúúv.Dw—¢ [¿î­Ø:béLHùAú;@øÞ/œ9–'î³`¦|~縧]\V>Þ,’«Õ¸#r ¸rš¢þ³ŸxœÿZÍ›wŒ™y¢òñ¦¥©jéüîpý³ÒüNo÷{ f†m<îwîã‹Ä^ÓóÓ°­|ªk%pî'‘˜}LÌJàÜÌäE±Ü;ÆêSÀ¸wàÜ;pîãÆYÄ€;wîpäp‡ ;Ç=³zý xé¥ëë*³>‚zÃ=ŸC½®ät§¯¯XŬk… Ž‰¯Èb¾w·®óîz}%õˆ»Ì‰;´†DüOëcB•æ앱·KO&ï ÞdJ'ÆÀú·$ÇbAN•S1¶ÔR‘˜ÖôÖ«$yEÂ4ç4µ‹;Ks"ªÕ]2{–Ê­"×n ÷x½¡Êi&¼ûLfQÔ{Îrš$9Z”˜ä[Ôk&óŠèœ#8(P'¸ëœˆÉú]½¶]Äç”ËIÄîñÆMs…©Ü Iì®’¶æ@Ù¯|½V1“o‘>;É+×u‡»Î‰(Y¤¤pgЫG¹ÎÈ÷î³aì‘È·ØéºûQîŸ| Øý÷ôÊL^N“$§"]ðIó-ŠßuuÌ”É+àÊ Ô!î…ëåÐážæ´„ ©ã1õ #µÀA64qïAÀ‚€;wê÷C~ƒ¢¼yûYÍò²Ò:åÛÞ_ËÖŸ£°gØûðé¬õ/´?æïÀx+©¸$àÜ;pîÀ¸wàÜ;pîÀ¸÷‚{rݽ½/´ßúêWcGk|Ÿ†vúü çIDATËÊqgâêdß®ý…–…¶gâÆ÷iø°¬¼÷š¹æ»|øXãöŒÖø^ ”•¬‰¶´ù….ŽÅäï÷X Ü;pîÀ¸ç[Øêº8“7¾WÇd%p¯u]i¤Æ÷Ž;Ÿ…cÆ}¼Æw> Ü[þBûãŒïÕðY Ü;pîÀ¸çšØöÚ>wa|¿¸ÆJ܉;"ïÈÊ“LX¶>Ñýlv2Þ~Ú ºwè.qøp‡ àAÀ‚†Ì:p‡€;wî4-ÜY ]S¾ý¶Ž½P—¸×úÕúå»hë¸wàwà÷Ñ6`¡[ ÜÇ‚û(Œº•À¸wàÜ»Tô쉖þyS ËGc.pîƒÆ=^z%bËÌÁ½¨|l¹<4¼îq‹Æ_Ș²ç¤%ã{å÷±â^­¾Bo°u¼{0÷y¼±{ÀÝÊóÄøóÏq·ŠÆ7ߘpŸ0îco—/IZÌ.zõc9ÿ/?\:DzòÚÃwŽ{I4–|V-ãáwKx÷ ã96ß.Q*âtÚ L/²lí#ãÄ»_(Ï·s¿cÜåøóNÇßß1f«SÎ,o¼”ÿüo«®½{éLhî׫ë¤Í¬;+yý±ÖüÎðzÃ=œQ(뽪~¥™©ˆÌÃ%@{ÂÕ îÊïÌ®œcRKì<¥ãoåÓÖ«éGÖG5þBë4v/-<î;÷î…3§×E®ß,,ÿeîGâ¼ÛîÅó;Ñɦ_k~˜ºÄ:¸ï·b$j¨£oòéʯŸ]™)/¿5\Þ1î¡)cñ güÉ?ÝËÊÇ+Ç»osq/*ŸÙß¡w?įլ¤þÍÆ—ãÎèx¿„êèï…‹¾›ñí]➎?Yu¨ÆÙµW'¦Óؽ¬üÆ=ßíãŠÈêÙãû³™ÍãÏÞ9î%åu{ÛÀ½|¾¶›û5æwkW2Ô î—/DRô½Õçvæ%±¸8æ긟y÷²òä#;Ç]FWË#omy4JõHÕgUV¡ü82u»Ã=&ÄäL(z>¿ŠZV>˜ÿs õ¸<_Ó×â*ÏïÖÌl–‘Z×ÝÅØûãùÌÍ/ŠÝËÊo»¤å4w¨o=z¡X\?ñŽ‡ïýÜØ=¯<ïúº»{Ù™Ðs2Ö‰Ïü¼±š[>”Ô{-à^>_K¢Œ\ã/×g‰÷âì@keþ¿È%3Ù¦!¨#Ü•ç7®ŠÝ™^7NkÜi]»Ê«RF„w12²Pf™µÝ­G•MQfÇ¿Gùtæ½Q–4“’”Ï(œáh@zwWS´"ó¨èL_”»¼¸šÉ2/ë¬7 «:3H¤ÊèlŠ:WN’â(‡åÒÙ£ ¨ïþìêØ]寤UëB‹C¨Ô»M"ûOæ6Q±½ÎBþÜ’×s¾ÎÒïR+÷'yRb‡5šf‚òXωÝCC?™@gpJòûzIÖ¬”û õò'Þ]s®XÖç‚L~ýYRRn"x‡ºÄ]zíØú¨yÖþø‹š±nÓ|æ‰wßòrSþD»$±»òàTžò¢oÒ¸>ÊäÿË–¤˜&ÄJ¨[ïžä)UQº‘Í=(¼ï¿¥Þ]}¢‚œã+3VÛëø\zqGþÅJgÎq˜Ê¨¯á¨üR¡œ@P‡¸Cp‡ àAÀ‚€;wê÷C~ƒ¢¼دz ý0Îý >ô¦f.ÎrCV÷-„Ñ}X\ëø1Sh%£Ù(‘#îC¦½Îí”ÖBKY[N¼÷š¬÷8·ãæ‡à䧌;(ë}wÖfÃǃüøp¢s¯Çúàót‘Ÿ0îlp¬÷x YÝ0øAÜ»b½çÓ8ëlè{ì÷!Ñ^‡õ1ÍÑ:sóÝ#Ü[f}W»6 Ô÷–YçƒPOp«Öü=ÂÈpíÃ;ˆ>rC¸·Áú°néûÈUuó]ôÚ¸xg½Zù¨ñ¡iŽj7*Üûtî•"ë|0ï¿'‰;ë‹u>P hQ‰æÖî`ÌÌ:ܦªÈ÷ÝluÊúÀQ îUÝ| =˦‡;ë–u>| ÕÆ{w÷t‹{/ÎçÎp¯èæì纸÷w3ëp¶OÜ›)•)rá,9¡ÜГiJUäYCÇuì¸gÓãò0)Ô'…{¡›g'®lTŒ÷&J%=uR0»Å&Ç:ŸâC ÎS6ËwÙa¬²»ŸÖ5QJÑ~ÞQ']ÅûøgÇQjñœt2¸WøóN¤k`Ó}Ë„[–Ñ“c[|õÔÃR͆ä“1Öç½À½17|Ë0©‚õ@vUS ŠÜêw€ûß¹{$X77N[Çî÷ßî×D÷Ô…¸OvåôÐ]4öŽŽ)оû>º“Ø0Wï,à>êã†AüáÞë"ÜÇÜ@ÌOkÍZ9pŸ@qYæ晿9÷É4Ô_ì à>¹Þë…ø*íîÓnàÝDpßP÷:¬ s4n)pG/Ó5˜€p‡ÐgÀîh pGËÐ@àŽ–¡è3´ DŸ¡eh ú -CÑghˆ>CËÐ@ôZÜÑgÀ½?EÝÔbç}olôZ6©†+O€m}<ÚÖ¸¯]ôZ6-ï¾ô rÃ?Ú†wGË&êÝ ïøÙå¡jÞÄ3<ùvÁùú7Æy²%^ áîc‡¥cP ecôîÜq#ÇæáÒvl ø /Þ̸Ø+·˜‰Ì}."œ`(вñÆî"n‘az¼|Ibwáïeì¾]ÈA!¶,7)Oç@–Ñ»ËÈ$^zòðî´ÒåOÆîÁ\n‰GÄ9Ìå]È1<@–7vM?Z’Ûöh;\ºÒƒ“wÿb'[ª˜*(вñÆàéñ Åîž¼N³U±{hÙr‹yD=•c sõP e£òî韙֌}\»ñ†Ñ?ö¸±ãõïte†¶þ bû@_™±úe€;xOîh pGËÐ@ôZ†¢ÏÐ24}†–µÛ@$·®Ý_À}ü‡Øß7é÷w®Ç³UK;aÚ] ÖcVr§Nzà¹áÀ½6&ã5G¸7tèà9ƒx¾p‡Z#@¸Cp‡ àAÀîè¸Cp‡ 1ëÿÚ‰8A«¨IEND®B`‚WebwareForPython-DBUtils-8de754c/pyproject.toml000066400000000000000000000102701524214146500216300ustar00rootroot00000000000000[build-system] build-backend = "setuptools.build_meta" requires = ["setuptools>=77"] [project] name = "DBUtils" version = "3.2.0" description = "Database connections for multi-threaded environments." license = "MIT" authors = [{name = "Christoph Zwerschke", email = "cito@online.de"}] requires-python = ">=3.7" classifiers = [ "Development Status :: 5 - Production/Stable", "Environment :: Web Environment", "Intended Audience :: Developers", "Operating System :: OS Independent", "Programming Language :: Python", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Topic :: Database", "Topic :: Internet :: WWW/HTTP :: Dynamic Content", "Topic :: Software Development :: Libraries :: Python Modules", ] [project.optional-dependencies] pg = ["PyGreSQL>=5"] docs = ["docutils"] tests = ["pytest>=7", "ruff"] [project.readme] file = "README.md" content-type = "text/markdown" [project.urls] Homepage = "https://webwareforpython.github.io/DBUtils/" Download = "https://pypi.org/project/DBUtils/" Documentation = "https://webwareforpython.github.io/DBUtils/main.html" Changelog = "https://webwareforpython.github.io/DBUtils/changelog.html" "Issue Tracker" = "https://github.com/WebwareForPython/DBUtils/issues" "Source Code" = "https://github.com/WebwareForPython/DBUtils" [tool.setuptools] packages = ["dbutils"] platforms = ["any"] include-package-data = false [tool.ruff] line-length = 79 target-version = "py37" [tool.ruff.lint] select = [ "A", # flake8-builtins # "ANN", # flake8-annotations "ARG", # flake8-unused-arguments "B", # flake8-bugbear # "BLE", # flake8-blind-except "C4", # flake8-comprehensions "C90", # McCabe cyclomatic complexity "COM", # flake8-commas "D", # pydocstyle "DTZ", # flake8-datetimez "E", # pycodestyle # "EM", # flake8-errmsg "ERA", # eradicate "EXE", # flake8-executable "F", # Pyflakes # "FBT", # flake8-boolean-trap "G", # flake8-logging-format "I", # isort "ICN", # flake8-import-conventions "INP", # flake8-no-pep420 "INT", # flake8-gettext "ISC", # flake8-implicit-str-concat "N", # pep8-naming "PGH", # pygrep-hooks "PIE", # flake8-pie "PL", # Pylint "PT", # flake8-pytest-style "PTH", # flake8-use-pathlib "PYI", # flake8-pyi # "Q", # flake8-quotes "RET", # flake8-return "RSE", # flake8-raise "RUF", # Ruff-specific rules "S", # flake8-bandit # "SLF", # flake8-self "SIM", # flake8-simplify "T10", # flake8-debugger "T20", # flake8-print "TCH", # flake8-type-checking "TID", # flake8-tidy-imports # "TRY", # tryceratops "UP", # pyupgrade "W", # pycodestyle "YTT", # flake8-2020 ] # Note: use `ruff rule ...` to see explanations of rules ignore = [ "D203", # no blank line before class docstring "D213", # multi-line docstrings should not start at second line "RUF022", # __all__ can have custom order ] [tool.ruff.lint.mccabe] max-complexity = 30 [tool.ruff.lint.flake8-quotes] inline-quotes = "double" [tool.ruff.lint.pylint] max-args = 12 max-branches = 35 max-statements = 95 [tool.ruff.lint.per-file-ignores] "docs/*" = [ "INP001", # allow stand-alone scripts "T201", # allow print statements ] "tests/*" = [ "D413", # allow plain notes in the module docstrings "PLC0415", # allow imports in tests functions "PLR2004", # allow magic values "S101", # allow assert statements ] "tests/mock_*.py" = [ # the mock objects only mimic a foreign API, docstrings would add nothing "D101", # missing docstring in public class "D102", # missing docstring in public method "D103", # missing docstring in public function "D105", # missing docstring in magic method "D107", # missing docstring in __init__ ] [tool.codespell] skip = '.git,.tox,.venv,*.de.html,*.de.rst,build,dist,local' quiet-level = 2 WebwareForPython-DBUtils-8de754c/tests/000077500000000000000000000000001524214146500200565ustar00rootroot00000000000000WebwareForPython-DBUtils-8de754c/tests/__init__.py000066400000000000000000000002161524214146500221660ustar00rootroot00000000000000"""The DBUtils tests package.""" import sys from . import mock_pg # make the mock pg module importable as "pg" sys.modules['pg'] = mock_pg WebwareForPython-DBUtils-8de754c/tests/conftest.py000066400000000000000000000023251524214146500222570ustar00rootroot00000000000000"""Fixtures for the DBUtils tests.""" import pytest from . import mock_db @pytest.fixture def dbapi(): """Get the mock DB-API 2 module.""" mock_db.threadsafety = 2 return mock_db @pytest.fixture def con_cls(): """Get the mock connection class with the ping counter reset. The ping method is unavailable, and the counter is reset again after the test, even when the test fails. """ con_cls = mock_db.Connection con_cls.has_ping = False con_cls.num_pings = 0 yield con_cls con_cls.has_ping = False con_cls.num_pings = 0 @pytest.fixture def ping_con_cls(con_cls): """Get the mock connection class with a working ping method.""" con_cls.has_ping = True return con_cls @pytest.fixture def pinger(con_cls): # noqa: ARG001 """Get a check function that can be passed as ping parameter. The connection class does not have a ping method, so that the check function must be used. It reports valid connections as alive, just as the ping method would do, and it records all of its calls. """ def check(con): check.calls.append(con) if not con.valid: raise mock_db.OperationalError check.calls = [] return check WebwareForPython-DBUtils-8de754c/tests/mock_db.py000066400000000000000000000074601524214146500220350ustar00rootroot00000000000000"""Mock object for the DB-API 2 module.""" threadsafety = 2 class Error(Exception): pass class DatabaseError(Error): pass class OperationalError(DatabaseError): pass class InterfaceError(DatabaseError): pass class InternalError(DatabaseError): pass class ProgrammingError(DatabaseError): pass def connect(database=None, user=None): return Connection(database, user) class Connection: has_ping = False num_pings = 0 # number of statements that ran into a deliberate server side timeout, # counted on the class since the failover may use a new connection num_timeouts = 0 def __init__(self, database=None, user=None): self.database = database self.user = user self.valid = False if database == 'error': raise OperationalError self.open_cursors = 0 self.num_uses = 0 self.num_queries = 0 self.num_pings = 0 self.session = [] self.valid = True def close(self): if not self.valid: raise InternalError self.open_cursors = 0 self.num_uses = 0 self.num_queries = 0 self.session = [] self.valid = False def commit(self): if not self.valid: raise InternalError self.session.append('commit') def rollback(self): if not self.valid: raise InternalError self.session.append('rollback') def ping(self): cls = self.__class__ cls.num_pings += 1 if not cls.has_ping: raise AttributeError if not self.valid: raise OperationalError def cursor(self, name=None): if not self.valid: raise InternalError return Cursor(self, name) class Cursor: def __init__(self, con, name=None): self.con = con self.valid = False if name == 'error': raise OperationalError self.result = None self.inputsizes = [] self.outputsizes = {} con.open_cursors += 1 self.valid = True def close(self): if not self.valid: raise InternalError self.con.open_cursors -= 1 self.valid = False def execute(self, operation): if not self.valid or not self.con.valid: raise InternalError self.con.num_uses += 1 if operation.startswith('select '): self.con.num_queries += 1 self.result = operation[7:] elif operation.startswith('set '): self.con.session.append(operation[4:]) self.result = None elif operation == 'timeout': # simulate a statement that has been aborted by the server # because it exceeded a deliberately requested time limit; # the connection itself stays perfectly usable type(self.con).num_timeouts += 1 raise OperationalError(3024, 'max execution time exceeded') elif operation == 'get sizes': self.result = (self.inputsizes, self.outputsizes) self.inputsizes = [] self.outputsizes = {} else: raise ProgrammingError def fetchone(self): if not self.valid: raise InternalError result = self.result self.result = None return result def callproc(self, procname): if not self.valid or not self.con.valid or not procname: raise InternalError self.con.num_uses += 1 def setinputsizes(self, sizes): if not self.valid: raise InternalError self.inputsizes = sizes def setoutputsize(self, size, column=None): if not self.valid: raise InternalError self.outputsizes[column] = size def __del__(self): if self.valid: self.close() WebwareForPython-DBUtils-8de754c/tests/mock_pg.py000066400000000000000000000044651524214146500220600ustar00rootroot00000000000000"""Mock object for the pg API module.""" class Error(Exception): pass class DatabaseError(Error): pass class InternalError(DatabaseError): pass class ProgrammingError(DatabaseError): pass def connect(*args, **kwargs): return PgConnection(*args, **kwargs) class PgConnection: """The underlying pg API connection class.""" def __init__(self, dbname=None, user=None): self.db = dbname self.user = user self.num_queries = 0 self.session = [] if dbname == 'error': self.status = False self.valid = False raise InternalError self.status = True self.valid = True def close(self): if not self.valid: raise InternalError self.num_queries = 0 self.session = [] self.status = False self.valid = False def reset(self): self.num_queries = 0 self.session = [] self.status = True self.valid = True def query(self, qstr): if not self.valid: raise InternalError if qstr in ('begin', 'end', 'commit', 'rollback'): self.session.append(qstr) return None if qstr.startswith('select '): self.num_queries += 1 return qstr[7:] if qstr.startswith('set '): self.session.append(qstr[4:]) return None raise ProgrammingError class DB: """Wrapper class for the pg API connection class.""" def __init__(self, *args, **kw): self.db = connect(*args, **kw) self.dbname = self.db.db self.__args = args, kw def __getattr__(self, name): if not self.db: raise AttributeError return getattr(self.db, name) def close(self): if not self.db: raise InternalError self.db.close() self.db = None def reopen(self): if self.db: self.close() try: self.db = connect(*self.__args[0], **self.__args[1]) except Exception: self.db = None raise def query(self, qstr): if not self.db: raise InternalError return self.db.query(qstr) def get_tables(self): if not self.db: raise InternalError return 'test' WebwareForPython-DBUtils-8de754c/tests/test_persistent_db.py000066400000000000000000000326731524214146500243470ustar00rootroot00000000000000"""Test the PersistentDB module. Note: We don't test performance here, so the test does not predicate whether PersistentDB actually will help in improving performance or not. We also assume that the underlying SteadyDB connections are tested. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pytest from dbutils.persistent_db import NotSupportedError, PersistentDB, local def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, persistent_db assert persistent_db.__version__ == __version__ assert PersistentDB.version == __version__ @pytest.mark.parametrize("threadsafety", [None, 0]) def test_no_threadsafety(dbapi, threadsafety): """Check that a database module that is not thread-safe is rejected.""" dbapi.threadsafety = threadsafety with pytest.raises(NotSupportedError): PersistentDB(dbapi) def test_creator_function(dbapi): """Check that a creator function can be used instead of a module.""" persist = PersistentDB(dbapi.connect, database='ok') # the threadsafety cannot be determined from a plain creator function, # so the connections are optimistically assumed to be thread-safe db = persist.connection() assert db.threadsafety() == dbapi.threadsafety cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_creator_without_threadsafety(dbapi): """Check that a creator that hides its threadsafety is rejected.""" class Creator: """A database module that does not report its threadsafety.""" connect = staticmethod(dbapi.connect) with pytest.raises(NotSupportedError): PersistentDB(Creator) def test_creator_with_unsafe_connections(dbapi, monkeypatch): """Check that connections that are not thread-safe are rejected.""" monkeypatch.delattr(dbapi, 'threadsafety') # a creator function is optimistically assumed to provide thread-safe # connections, but the connections themselves know better persist = PersistentDB(dbapi.connect) with pytest.raises(NotSupportedError): persist.connection() @pytest.mark.parametrize("closeable", [False, True]) def test_close(dbapi, closeable): """Check that closing is only allowed when the connection is closeable.""" persist = PersistentDB(dbapi, closeable=closeable) db = persist.connection() assert db._con.valid is True db.close() assert closeable ^ db._con.valid db.close() assert closeable ^ db._con.valid db._close() assert db._con.valid is False db._close() assert db._con.valid is False def test_connection(dbapi): """Check that the same thread always gets the same connection.""" persist = PersistentDB(dbapi) db = persist.connection() db_con = db._con assert db_con.database is None assert db_con.user is None db2 = persist.connection() assert db == db2 db3 = persist.dedicated_connection() assert db == db3 db3.close() db2.close() db.close() def test_threads(dbapi): """Check that every thread keeps its own persistent connection.""" num_threads = 3 persist = PersistentDB(dbapi, closeable=True) query_queue, result_queue = [], [] for _i in range(num_threads): query_queue.append(Queue(1)) result_queue.append(Queue(1)) def run_queries(idx): """Answer the queries sent to this thread until it gets None. Every answer is prefixed with the thread number and the usage count of the connection, so that the main thread can check that the thread kept using its own connection. """ this_db = persist.connection() db = None while True: try: q = query_queue[idx].get(timeout=1) except Empty: q = None if not q: break db = persist.connection() if db != this_db: res = 'error - not persistent' elif q == 'ping': res = 'ok - thread alive' elif q == 'close': db.close() res = 'ok - connection closed' else: cursor = db.cursor() cursor.execute(q) res = cursor.fetchone() cursor.close() res = f'{idx}({db._usage}): {res}' result_queue[idx].put(res, timeout=1) if db: db.close() threads = [] for i in range(num_threads): thread = Thread(target=run_queries, args=(i,)) threads.append(thread) thread.start() # all threads are alive and have an unused connection of their own for i in range(num_threads): query_queue[i].put('ping', timeout=1) for i in range(num_threads): r = result_queue[i].get(timeout=1) assert r == f'{i}(0): ok - thread alive' assert threads[i].is_alive() # let thread number i run i + 1 queries on its own connection for i in range(num_threads): for j in range(i + 1): query_queue[i].put(f'select test{j}', timeout=1) r = result_queue[i].get(timeout=1) assert r == f'{i}({j + 1}): test{j}' # closing the connection of the second thread restarts its usage count, # but does not affect the connections of the other threads query_queue[1].put('select test4', timeout=1) r = result_queue[1].get(timeout=1) assert r == '1(3): test4' query_queue[1].put('close', timeout=1) r = result_queue[1].get(timeout=1) assert r == '1(3): ok - connection closed' for j in range(2): query_queue[1].put(f'select test{j}', timeout=1) r = result_queue[1].get(timeout=1) assert r == f'1({j + 1}): test{j}' # every thread still has its own connection with its own usage count for i in range(num_threads): assert threads[i].is_alive() query_queue[i].put('ping', timeout=1) for i in range(num_threads): r = result_queue[i].get(timeout=1) assert r == f'{i}({i + 1}): ok - thread alive' assert threads[i].is_alive() # send the sentinel that makes the threads finish for i in range(num_threads): query_queue[i].put(None, timeout=1) def test_maxusage(dbapi): """Check that the connection is reset when used too often.""" persist = PersistentDB(dbapi, 20) db = persist.connection() assert db._maxusage == 20 for i in range(100): cursor = db.cursor() cursor.execute(f'select test{i}') r = cursor.fetchone() cursor.close() assert r == f'test{i}' assert db._con.valid is True j = i % 20 + 1 assert db._usage == j assert db._con.num_uses == j assert db._con.num_queries == j def test_setsession(dbapi): """Check that the session is prepared after every reopening.""" persist = PersistentDB(dbapi, 3, ('set datestyle',)) db = persist.connection() assert db._maxusage == 3 assert db._setsession_sql == ('set datestyle',) assert db._con.session == ['datestyle'] cursor = db.cursor() cursor.execute('set test') cursor.fetchone() cursor.close() for _i in range(3): assert db._con.session == ['datestyle', 'test'] cursor = db.cursor() cursor.execute('select test') cursor.fetchone() cursor.close() assert db._con.session == ['datestyle'] def test_threadlocal(dbapi): """Check that the class for thread-local data can be replaced.""" persist = PersistentDB(dbapi) assert isinstance(persist.thread, local) class Threadlocal: pass persist = PersistentDB(dbapi, threadlocal=Threadlocal) assert isinstance(persist.thread, Threadlocal) def test_ping_check_never(dbapi, ping_con_cls): """Check that connections are not pinged when ping is 0.""" persist = PersistentDB(dbapi, 0, None, None, 0, True) db = persist.connection() assert db._con.valid is True assert ping_con_cls.num_pings == 0 # the closed connection is handed out again without being pinged, # so the fact that it is broken goes unnoticed db.close() db = persist.connection() assert db._con.valid is False assert ping_con_cls.num_pings == 0 def test_ping_check_when_fetched(dbapi, ping_con_cls): """Check that connections are pinged when ping is 1.""" persist = PersistentDB(dbapi, 0, None, None, 1, True) db = persist.connection() assert db._con.valid is True assert ping_con_cls.num_pings == 1 # the closed connection is pinged when it is requested again, # so it is transparently reopened db.close() db = persist.connection() assert db._con.valid is True assert ping_con_cls.num_pings == 2 def test_ping_check_with_sql_when_fetched(dbapi, con_cls): """Check that connections can be checked with an SQL statement.""" persist = PersistentDB(dbapi, 0, None, None, 'select 1', True) db = persist.connection() assert db._con.valid is True # the statement has been executed instead of using the ping method assert db._con.num_queries == 1 assert con_cls.num_pings == 0 # the closed connection is checked when it is requested again, # so it is transparently reopened db.close() db = persist.connection() assert db._con.valid is True assert con_cls.num_pings == 0 def test_ping_check_when_cursor_created(dbapi, ping_con_cls): """Check that connections are pinged when ping is 2.""" persist = PersistentDB(dbapi, 0, None, None, 2, True) db = persist.connection() assert db._con.valid is True assert ping_con_cls.num_pings == 0 # requesting the closed connection does not ping it yet db.close() db = persist.connection() assert db._con.valid is False assert ping_con_cls.num_pings == 0 # but creating a cursor does, which reopens the connection cursor = db.cursor() assert db._con.valid is True assert ping_con_cls.num_pings == 1 # executing a query does not ping again cursor.execute('select test') assert db._con.valid is True assert ping_con_cls.num_pings == 1 def test_ping_check_when_query_executed(dbapi, ping_con_cls): """Check that connections are pinged when ping is 4.""" persist = PersistentDB(dbapi, 0, None, None, 4, True) db = persist.connection() assert db._con.valid is True assert ping_con_cls.num_pings == 0 # neither requesting the connection nor creating a cursor pings it db.close() db = persist.connection() assert db._con.valid is False assert ping_con_cls.num_pings == 0 cursor = db.cursor() db._con.close() assert db._con.valid is False assert ping_con_cls.num_pings == 0 # only executing a query does, which reopens the connection cursor.execute('select test') assert db._con.valid is True assert ping_con_cls.num_pings == 1 def test_failed_transaction(dbapi): """Check that a failed transaction is reported and recovered from.""" persist = PersistentDB(dbapi) db = persist.connection() cursor = db.cursor() db._con.close() cursor.execute('select test') db.begin() db._con.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test') cursor.execute('select test') db.begin() db.cancel() db._con.close() cursor.execute('select test') def test_context_manager(dbapi): """Check that connection and cursor can be used as context managers.""" persist = PersistentDB(dbapi) with persist.connection() as db: with db.cursor() as cursor: cursor.execute('select test') r = cursor.fetchone() assert r == 'test' def timeout_is_not_fatal(error): """Treat a deliberate server side timeout as not fatal.""" return not error.args or error.args[0] != 3024 def test_isfatal_default(dbapi): """Check that no error check is configured by default.""" persist = PersistentDB(dbapi) assert persist._isfatal is None db = persist.connection() assert db._isfatal is None def test_isfatal(dbapi): """Check that isfatal is passed on and can veto the failover.""" persist = PersistentDB(dbapi, isfatal=timeout_is_not_fatal) assert persist._isfatal is timeout_is_not_fatal db = persist.connection() assert db._isfatal is timeout_is_not_fatal dbapi.Connection.num_timeouts = 0 cursor = db.cursor() with pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 # the failover mechanism still applies to other failures db._con.close() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_no_failover(dbapi): """Check that no_failover() suspends the failover mechanism.""" persist = PersistentDB(dbapi) db = persist.connection() dbapi.Connection.num_timeouts = 0 with db.cursor() as cursor: with cursor.no_failover(), pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 cursor.execute('select test') # connection is still usable assert cursor.fetchone() == 'test' def test_dbapi_connection(dbapi): """Check that the underlying DB-API 2 objects are accessible.""" persist = PersistentDB(dbapi) db = persist.connection() con = db.dbapi_connection assert isinstance(con, dbapi.Connection) assert con is db._con cursor = db.cursor() assert cursor.dbapi_cursor is cursor._cursor WebwareForPython-DBUtils-8de754c/tests/test_persistent_pg.py000066400000000000000000000135411524214146500243610ustar00rootroot00000000000000"""Test the PersistentPg module. Note: We don't test performance here, so the test does not predicate whether PersistentPg actually will help in improving performance or not. We also assume that the underlying SteadyPg connections are tested. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pg import pytest from dbutils.persistent_pg import PersistentPg def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, persistent_pg assert persistent_pg.__version__ == __version__ assert PersistentPg.version == __version__ @pytest.mark.parametrize("closeable", [False, True]) def test_close(closeable): """Check that closing is only allowed when the connection is closeable.""" persist = PersistentPg(closeable=closeable) db = persist.connection() assert db._con.db assert db._con.valid is True db.close() assert closeable ^ (db._con.db is not None and db._con.valid) db.close() assert closeable ^ (db._con.db is not None and db._con.valid) db._close() assert not db._con.db db._close() assert not db._con.db def test_threads(): """Check that every thread keeps its own persistent connection.""" num_threads = 3 persist = PersistentPg() query_queue, result_queue = [], [] for _i in range(num_threads): query_queue.append(Queue(1)) result_queue.append(Queue(1)) def run_queries(idx): """Answer the queries sent to this thread until it gets None. Every answer is prefixed with the thread number and the usage count of the connection, so that the main thread can check that the thread kept using its own connection. """ this_db = persist.connection().db db = None while True: try: q = query_queue[idx].get(timeout=1) except Empty: q = None if not q: break db = persist.connection() if db.db != this_db: res = 'error - not persistent' elif q == 'ping': res = 'ok - thread alive' elif q == 'close': db.db.close() res = 'ok - connection closed' else: res = db.query(q) res = f'{idx}({db._usage}): {res}' result_queue[idx].put(res, timeout=1) if db: db.close() threads = [] for i in range(num_threads): thread = Thread(target=run_queries, args=(i,)) threads.append(thread) thread.start() # all threads are alive and have an unused connection of their own for i in range(num_threads): query_queue[i].put('ping', timeout=1) for i in range(num_threads): r = result_queue[i].get(timeout=1) assert r == f'{i}(0): ok - thread alive' assert threads[i].is_alive() # let thread number i run i + 1 queries on its own connection for i in range(num_threads): for j in range(i + 1): query_queue[i].put(f'select test{j}', timeout=1) r = result_queue[i].get(timeout=1) assert r == f'{i}({j + 1}): test{j}' # closing the connection of the second thread restarts its usage count, # but does not affect the connections of the other threads query_queue[1].put('select test4', timeout=1) r = result_queue[1].get(timeout=1) assert r == '1(3): test4' query_queue[1].put('close', timeout=1) r = result_queue[1].get(timeout=1) assert r == '1(3): ok - connection closed' for j in range(2): query_queue[1].put(f'select test{j}', timeout=1) r = result_queue[1].get(timeout=1) assert r == f'1({j + 1}): test{j}' # every thread still has its own connection with its own usage count for i in range(num_threads): assert threads[i].is_alive() query_queue[i].put('ping', timeout=1) for i in range(num_threads): r = result_queue[i].get(timeout=1) assert r == f'{i}({i + 1}): ok - thread alive' assert threads[i].is_alive() # send the sentinel that makes the threads finish for i in range(num_threads): query_queue[i].put(None, timeout=1) def test_maxusage(): """Check that the connection is reset when used too often.""" persist = PersistentPg(20) db = persist.connection() assert db._maxusage == 20 for i in range(100): r = db.query(f'select test{i}') assert r == f'test{i}' assert db.db.status j = i % 20 + 1 assert db._usage == j assert db.num_queries == j def test_setsession(): """Check that the session is prepared after every reopening.""" persist = PersistentPg(3, ('set datestyle',)) db = persist.connection() assert db._maxusage == 3 assert db._setsession_sql == ('set datestyle',) assert db.db.session == ['datestyle'] db.query('set test') for _i in range(3): assert db.db.session == ['datestyle', 'test'] db.query('select test') assert db.db.session == ['datestyle'] def test_failed_transaction(): """Check that a failed transaction is reported and recovered from.""" persist = PersistentPg() db = persist.connection() db._con.close() assert db.query('select test') == 'test' db.begin() db._con.close() with pytest.raises(pg.InternalError): db.query('select test') assert db.query('select test') == 'test' db.begin() assert db.query('select test') == 'test' db.rollback() db._con.close() assert db.query('select test') == 'test' def test_context_manager(): """Check that the connection can be used as a context manager.""" persist = PersistentPg() with persist.connection() as db: db.query('select test') assert db.num_queries == 1 WebwareForPython-DBUtils-8de754c/tests/test_pooled_db.py000066400000000000000000001607401524214146500234260ustar00rootroot00000000000000"""Test the PooledDB module. Note: We don't test performance here, so the test does not predicate whether PooledDB actually will help in improving performance or not. We also assume that the underlying SteadyDB connections are tested. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pytest from dbutils.pooled_db import ( InvalidConnectionError, NotSupportedError, PooledDB, SharedDBConnection, TooManyConnectionsError, ) from dbutils.steady_db import SteadyDBConnection def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, pooled_db assert pooled_db.__version__ == __version__ assert PooledDB.version == __version__ @pytest.mark.parametrize("threadsafety", [None, 0]) def test_no_threadsafety(dbapi, threadsafety): """Check that a database module that is not thread-safe is rejected.""" dbapi.threadsafety = threadsafety with pytest.raises(NotSupportedError): PooledDB(dbapi) def test_creator_function(dbapi): """Check that a creator function can be used instead of a module.""" pool = PooledDB(dbapi.connect, 1, database='ok') # the threadsafety cannot be determined from a plain creator function, # so the connections are optimistically assumed to be thread-safe, # but not safe enough to be shared between several threads assert pool._maxshared == 0 db = pool.connection() assert db.threadsafety() == dbapi.threadsafety cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_creator_without_threadsafety(dbapi): """Check that a creator that hides its threadsafety is rejected.""" class Creator: """A database module that does not report its threadsafety.""" connect = staticmethod(dbapi.connect) with pytest.raises(NotSupportedError): PooledDB(Creator) def test_creator_with_unsafe_connections(dbapi, monkeypatch): """Check that connections that are not thread-safe are rejected.""" monkeypatch.delattr(dbapi, 'threadsafety') # the pool optimistically assumes that a creator function provides # thread-safe connections, but the connections themselves know better pool = PooledDB(dbapi.connect, 0) with pytest.raises(NotSupportedError): pool.connection() @pytest.mark.parametrize("threadsafety", [1, 2, 3]) def test_threadsafety(dbapi, threadsafety): """Check that connections are only shared when they may be.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, 0, 0, 1) assert hasattr(pool, '_maxshared') if threadsafety > 1: assert pool._maxshared == 1 assert hasattr(pool, '_shared_cache') else: assert pool._maxshared == 0 assert not hasattr(pool, '_shared_cache') @pytest.mark.parametrize("threadsafety", [1, 2]) def test_create_connection(dbapi, threadsafety): """Check that the pool creates connections with the given parameters.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB( dbapi, 1, 1, 1, 0, False, None, None, True, None, None, None, 'PooledDBTestDB', user='PooledDBTestUser') # mincached=1, so one connection has been created up front assert hasattr(pool, '_idle_cache') assert len(pool._idle_cache) == 1 if shareable: assert hasattr(pool, '_shared_cache') assert len(pool._shared_cache) == 0 else: assert not hasattr(pool, '_shared_cache') assert hasattr(pool, '_maxusage') assert pool._maxusage is None assert hasattr(pool, '_setsession') assert pool._setsession is None # the cached connection is a steady connection without extra settings con = pool._idle_cache[0] assert isinstance(con, SteadyDBConnection) assert hasattr(con, '_maxusage') assert con._maxusage == 0 assert hasattr(con, '_setsession_sql') assert con._setsession_sql is None # taking it out of the pool moves it from the idle to the shared cache db = pool.connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 assert hasattr(db, '_con') assert db._con == con assert hasattr(db, 'cursor') assert hasattr(db, '_usage') assert db._usage == 0 # the connect parameters have been passed down to the DB-API 2 connection assert hasattr(con, '_con') db_con = con._con assert hasattr(db_con, 'database') assert db_con.database == 'PooledDBTestDB' assert hasattr(db_con, 'user') assert db_con.user == 'PooledDBTestUser' assert hasattr(db_con, 'open_cursors') assert db_con.open_cursors == 0 assert hasattr(db_con, 'num_uses') assert db_con.num_uses == 0 assert hasattr(db_con, 'num_queries') assert db_con.num_queries == 0 # a query opens and closes a cursor and counts as one query and one use cursor = db.cursor() assert db_con.open_cursors == 1 cursor.execute('select test') r = cursor.fetchone() cursor.close() assert db_con.open_cursors == 0 assert r == 'test' assert db_con.num_queries == 1 assert db._usage == 1 # two cursors can be open at the same time on the same connection, # and a "set" command is recorded in the session instead of as a query cursor = db.cursor() assert db_con.open_cursors == 1 cursor.execute('set sessiontest') cursor2 = db.cursor() assert db_con.open_cursors == 2 cursor2.close() assert db_con.open_cursors == 1 cursor.close() assert db_con.open_cursors == 0 assert db_con.num_queries == 1 assert db._usage == 2 assert db_con.session == ['rollback', 'sessiontest'] @pytest.mark.parametrize("threadsafety", [1, 2]) def test_dedicated_connection(dbapi, threadsafety): """Check the difference between shared and dedicated connections.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 1, 1) assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 # connection() defaults to a shared connection db = pool.connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 db.close() assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 # asking for a shared connection explicitly does the same db = pool.connection(True) assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 db.close() assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 # a dedicated connection never enters the shared cache db = pool.connection(False) assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 assert db._usage == 0 db_con = db._con._con assert db_con.database is None assert db_con.user is None db.close() assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 # dedicated_connection() is a shortcut for connection(False) db = pool.dedicated_connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 assert db._usage == 0 db_con = db._con._con assert db_con.database is None assert db_con.user is None db.close() assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 def test_create_connection_with_settings(dbapi): """Check that maxusage and setsession are passed on to the connections.""" pool = PooledDB(dbapi, 0, 0, 0, 0, False, 3, ('set datestyle',)) assert pool._maxusage == 3 assert pool._setsession == ('set datestyle',) con = pool.connection()._con assert con._maxusage == 3 assert con._setsession_sql == ('set datestyle',) @pytest.mark.parametrize("threadsafety", [1, 2]) def test_close_connection(dbapi, threadsafety): """Check that a closed connection is returned to the pool.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB( dbapi, 0, 1, 1, 0, False, None, None, True, None, None, None, 'PooledDBTestDB', user='PooledDBTestUser') assert hasattr(pool, '_idle_cache') assert len(pool._idle_cache) == 0 # take a connection out of the pool and note its underlying connection db = pool.connection() assert hasattr(db, '_con') con = db._con assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 assert hasattr(db, '_shared_con') shared_con = db._shared_con assert pool._shared_cache[0] == shared_con assert hasattr(shared_con, 'shared') assert shared_con.shared == 1 assert hasattr(shared_con, 'con') assert shared_con.con == con assert isinstance(con, SteadyDBConnection) assert hasattr(con, '_con') db_con = con._con assert hasattr(db_con, 'num_queries') assert db._usage == 0 assert db_con.num_queries == 0 db.cursor().execute('select test') assert db._usage == 1 assert db_con.num_queries == 1 # closing the pooled connection invalidates it, but returns the # underlying connection to the idle cache instead of closing it db.close() assert db._con is None if shareable: assert db._shared_con is None assert shared_con.shared == 0 with pytest.raises(InvalidConnectionError): assert db._usage assert not hasattr(db_con, '_num_queries') assert len(pool._idle_cache) == 1 assert pool._idle_cache[0]._con == db_con if shareable: assert len(pool._shared_cache) == 0 # closing it a second time is a no-op db.close() if shareable: assert shared_con.shared == 0 # the next request gets the same underlying connection back, # with its usage counters and connect parameters still intact db = pool.connection() assert db._con == con assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 shared_con = db._shared_con assert pool._shared_cache[0] == shared_con assert shared_con.con == con assert shared_con.shared == 1 assert db._usage == 1 assert db_con.num_queries == 1 assert hasattr(db_con, 'database') assert db_con.database == 'PooledDBTestDB' assert hasattr(db_con, 'user') assert db_con.user == 'PooledDBTestUser' db.cursor().execute('select test') assert db_con.num_queries == 2 db.cursor().execute('select test') assert db_con.num_queries == 3 db.close() assert len(pool._idle_cache) == 1 assert pool._idle_cache[0]._con == db_con if shareable: assert len(pool._shared_cache) == 0 # a dedicated connection is returned to the idle cache as well db = pool.connection(False) assert db._con == con assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 db.close() assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 def test_close_all(dbapi): """Check that closing the pool empties the idle cache.""" pool = PooledDB(dbapi, 10) assert len(pool._idle_cache) == 10 pool.close() assert len(pool._idle_cache) == 0 def test_close_all_when_pool_is_deleted(dbapi): """Check that deleting the pool closes the cached connections.""" pool = PooledDB(dbapi, 10) closed = ['no'] def close(what=closed): what[0] = 'yes' pool._idle_cache[7]._con.close = close assert closed == ['no'] del pool assert closed == ['yes'] @pytest.mark.parametrize("threadsafety", [1, 2]) def test_close_all_shared(dbapi, threadsafety): """Check that closing the pool empties the shared cache as well.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 10, 10, 5) assert len(pool._idle_cache) == 10 if shareable: assert len(pool._shared_cache) == 0 cache = [] for _i in range(5): cache.append(pool.connection()) assert len(pool._idle_cache) == 5 if shareable: assert len(pool._shared_cache) == 5 pool.close() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_close_all_shared_when_pool_is_deleted(dbapi, threadsafety): """Check that deleting the pool closes idle and shared connections.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 10, 10, 5) closed = [] def close_idle(what=closed): what.append('idle') def close_shared(what=closed): what.append('shared') # spy on one idle and one shared connection (or, when connections # cannot be shared, on a second idle connection closed after the first) if shareable: cache = [] for _i in range(5): cache.append(pool.connection()) pool._shared_cache[3].con.close = close_shared else: pool._idle_cache[7]._con.close = close_shared pool._idle_cache[3]._con.close = close_idle assert closed == [] del pool if shareable: del cache assert closed == ['idle', 'shared'] @pytest.mark.parametrize("threadsafety", [1, 2]) def test_shareable_connection(dbapi, threadsafety): """Check that connections are shared when the maximum is reached.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 1, 2) assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 # the first two requests get connections of their own db1 = pool.connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 db2 = pool.connection() assert db1._con != db2._con assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 2 # maxshared is reached now, so the next requests share those connections db3 = pool.connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 2 assert db3._con == db1._con assert db1._shared_con.shared == 2 assert db2._shared_con.shared == 1 else: assert db3._con != db1._con assert db3._con != db2._con db4 = pool.connection() assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 2 assert db4._con == db2._con assert db1._shared_con.shared == 2 assert db2._shared_con.shared == 2 else: assert db4._con != db1._con assert db4._con != db2._con assert db4._con != db3._con # a dedicated connection is never taken from the shared cache db5 = pool.connection(False) assert db5._con != db1._con assert db5._con != db2._con assert db5._con != db3._con assert db5._con != db4._con assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 2 assert db1._shared_con.shared == 2 assert db2._shared_con.shared == 2 # and once returned it stays in the idle cache, # since a shareable request prefers to share an existing connection db5.close() assert len(pool._idle_cache) == 1 db5 = pool.connection() if shareable: assert len(pool._idle_cache) == 1 assert len(pool._shared_cache) == 2 assert db5._shared_con.shared == 3 else: assert len(pool._idle_cache) == 0 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_unshare_connection(dbapi, threadsafety): """Check that a shared connection is cached again when fully unshared.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 0, 1) assert len(pool._idle_cache) == 0 db1 = pool.connection(False) if shareable: assert len(pool._shared_cache) == 0 db2 = pool.connection() if shareable: assert len(pool._shared_cache) == 1 db3 = pool.connection() if shareable: assert len(pool._shared_cache) == 1 assert db2._con == db3._con else: assert db2._con != db3._con # dropping one of the two sharers leaves the connection shared once, # so it is not cached as idle yet del db3 if shareable: assert len(pool._idle_cache) == 0 assert len(pool._shared_cache) == 1 else: assert len(pool._idle_cache) == 1 # dropping the last sharer moves it back to the idle cache del db2 if shareable: assert len(pool._idle_cache) == 1 assert len(pool._shared_cache) == 0 else: assert len(pool._idle_cache) == 2 del db1 if shareable: assert len(pool._idle_cache) == 2 assert len(pool._shared_cache) == 0 else: assert len(pool._idle_cache) == 3 @pytest.mark.parametrize("threadsafety", [1, 2]) @pytest.mark.parametrize(("mincached", "maxcached", "cached", "rounds"), [ # every round takes the given number of connections out of the pool # and expects the given number of idle connections after releasing them (3, 0, 3, [(3, 3), (6, 6)]), # unlimited cache grows with the demand (3, None, 3, [(3, 3), (6, 6)]), # None is the same as zero here (None, None, 0, [(3, 3), (6, 6)]), # and also when nothing is cached (0, 3, 0, [(3, 3), (6, 3)]), # cache fills up to maxcached (3, 3, 3, [(3, 3), (6, 3)]), # cache stays at the common bound (3, 2, 3, [(4, 3)]), # mincached wins when it exceeds maxcached (2, 5, 2, [(10, 5)]), # cache grows from mincached to maxcached ]) def test_min_max_cached(dbapi, threadsafety, mincached, maxcached, cached, rounds): """Check that the idle cache is kept between the given bounds.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, mincached, maxcached) assert len(pool._idle_cache) == cached for taken, cached_again in rounds: cache = [pool.connection() for _i in range(taken)] assert len(pool._idle_cache) == 0 assert cache del cache assert len(pool._idle_cache) == cached_again @pytest.mark.parametrize("threadsafety", [1, 2]) def test_min_max_cached_with_shared(dbapi, threadsafety): """Check the idle cache bounds when connections can also be shared.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 2, 3) assert len(pool._idle_cache) == 1 # dedicated connections are cached up to maxcached when released cache = [pool.connection(False) for _i in range(4)] assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 assert cache del cache assert len(pool._idle_cache) == 2 # shared connections do not touch the idle cache while they are in use cache = [pool.connection() for _i in range(10)] assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 3 assert cache del cache assert len(pool._idle_cache) == 2 if shareable: assert len(pool._shared_cache) == 0 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_min_max_cached_with_fewer_shared(dbapi, threadsafety): """Check the idle cache bounds when maxshared is below maxcached.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 3, 2) assert len(pool._idle_cache) == 1 cache = [pool.connection(False) for _i in range(4)] assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 assert cache del cache assert len(pool._idle_cache) == 3 # only maxshared connections are needed, so one stays in the idle cache cache = [pool.connection() for _i in range(10)] if shareable: assert len(pool._idle_cache) == 1 assert len(pool._shared_cache) == 2 else: assert len(pool._idle_cache) == 0 assert cache del cache assert len(pool._idle_cache) == 3 if shareable: assert len(pool._shared_cache) == 0 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_max_shared(dbapi, threadsafety): """Check that the maximum number of shared connections is not exceeded.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi) assert len(pool._idle_cache) == 0 cache = [pool.connection() for _i in range(10)] assert len(cache) == 10 assert len(pool._idle_cache) == 0 pool = PooledDB(dbapi, 1, 1, 0) assert len(pool._idle_cache) == 1 cache = [pool.connection() for _i in range(10)] assert len(cache) == 10 assert len(pool._idle_cache) == 0 pool = PooledDB(dbapi, 0, 0, 1) cache = [pool.connection() for _i in range(10)] assert len(cache) == 10 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 pool = PooledDB(dbapi, 1, 1, 1) assert len(pool._idle_cache) == 1 cache = [pool.connection() for _i in range(10)] assert len(cache) == 10 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 pool = PooledDB(dbapi, 0, 0, 7) cache = [pool.connection(False) for _i in range(3)] assert len(cache) == 3 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 cache = [pool.connection() for _i in range(10)] assert len(cache) == 10 assert len(pool._idle_cache) == 3 if shareable: assert len(pool._shared_cache) == 7 def test_sort_shared(dbapi): """Check that shared connections are sorted by transaction and shares.""" pool = PooledDB(dbapi, 0, 4, 4) cache = [] for _i in range(6): db = pool.connection() db.cursor().execute('select test') cache.append(db) for i, db in enumerate(cache): assert db._shared_con.shared == 1 if 2 <= i < 4 else 2 cache[2].begin() cache[3].begin() db = pool.connection() assert db._con is cache[0]._con db.close() cache[3].rollback() db = pool.connection() assert db._con is cache[3]._con @pytest.mark.parametrize("threadsafety", [1, 2]) def test_equally_shared(dbapi, threadsafety): """Check that the load is spread equally over the connections.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 5, 5, 5) assert len(pool._idle_cache) == 5 for _i in range(15): db = pool.connection(False) db.cursor().execute('select test') db.close() assert len(pool._idle_cache) == 5 for i in range(5): con = pool._idle_cache[i] assert con._usage == 3 assert con._con.num_queries == 3 cache = [] for _i in range(35): db = pool.connection() db.cursor().execute('select test') cache.append(db) del db assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 5 for i in range(5): con = pool._shared_cache[i] assert con.shared == 7 con = con.con assert con._usage == 10 assert con._con.num_queries == 10 del cache assert len(pool._idle_cache) == 5 if shareable: assert len(pool._shared_cache) == 0 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_many_shared(dbapi, threadsafety): """Check that many shared connections are counted correctly.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 0, 5) # 35 requests are spread evenly over the 5 shareable connections, # each of which then carries 7 shares and 3 uses per share cache = [] for _i in range(35): db = pool.connection() db.cursor().execute('select test1') db.cursor().execute('select test2') db.cursor().callproc('test3') cache.append(db) del db assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 5 for i in range(5): con = pool._shared_cache[i] assert con.shared == 7 con = con.con assert con._usage == 21 assert con._con.num_queries == 14 # dropping 6 of the requests unshares the connections unevenly, # and re-sorts the cache so that the least shared one comes first cache[3] = cache[8] = cache[33] = None cache[12] = cache[17] = cache[34] = None assert len(pool._shared_cache) == 5 assert pool._shared_cache[0].shared == 7 assert pool._shared_cache[1].shared == 7 assert pool._shared_cache[2].shared == 5 assert pool._shared_cache[3].shared == 4 assert pool._shared_cache[4].shared == 6 # 6 new requests fill up the gaps, so the load is balanced again for db in cache: if db: db.cursor().callproc('test4') for _i in range(6): db = pool.connection() db.cursor().callproc('test4') cache.append(db) del db for i in range(5): con = pool._shared_cache[i] assert con.shared == 7 con = con.con assert con._usage == 28 assert con._con.num_queries == 14 del cache if shareable: assert len(pool._idle_cache) == 5 assert len(pool._shared_cache) == 0 else: assert len(pool._idle_cache) == 35 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_rollback(dbapi, threadsafety): """Check that connections are rolled back when returned to the pool.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, 0, 1) assert len(pool._idle_cache) == 0 db = pool.connection(False) assert len(pool._idle_cache) == 0 assert db._con._con.open_cursors == 0 cursor = db.cursor() assert db._con._con.open_cursors == 1 cursor.execute('set doit1') db.commit() cursor.execute('set dont1') cursor.close() assert db._con._con.open_cursors == 0 del db assert len(pool._idle_cache) == 1 db = pool.connection(False) assert len(pool._idle_cache) == 0 assert db._con._con.open_cursors == 0 cursor = db.cursor() assert db._con._con.open_cursors == 1 cursor.execute('set doit2') cursor.close() assert db._con._con.open_cursors == 0 db.commit() session = db._con._con.session db.close() assert session == [ 'doit1', 'commit', 'dont1', 'rollback', 'doit2', 'commit', 'rollback'] @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections(dbapi, threadsafety): """Check that the maximum number of connections is enforced.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 2, 2, 3) assert hasattr(pool, '_maxconnections') assert pool._maxconnections == 3 assert hasattr(pool, '_connections') assert pool._connections == 0 assert len(pool._idle_cache) == 1 # three dedicated connections exhaust the pool cache = [] for _i in range(3): cache.append(pool.connection(False)) assert pool._connections == 3 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 with pytest.raises(TooManyConnectionsError): pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() # releasing them frees the pool again (only maxcached=2 are kept) cache = [] assert pool._connections == 0 assert len(pool._idle_cache) == 2 if shareable: assert len(pool._shared_cache) == 0 # three shareable connections only need maxshared=2 connections, # which leaves room for one more dedicated connection for _i in range(3): cache.append(pool.connection()) assert len(pool._idle_cache) == 0 if shareable: assert pool._connections == 2 assert len(pool._shared_cache) == 2 cache.append(pool.connection(False)) assert pool._connections == 3 assert len(pool._shared_cache) == 2 else: assert pool._connections == 3 with pytest.raises(TooManyConnectionsError): pool.connection(False) # further shareable requests are always served by sharing if shareable: cache.append(pool.connection(True)) assert pool._connections == 3 else: with pytest.raises(TooManyConnectionsError): pool.connection() del cache assert pool._connections == 0 assert len(pool._idle_cache) == 2 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_one(dbapi, threadsafety): """Check that a pool can be limited to a single connection.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 1, 1, 1) assert pool._maxconnections == 1 assert pool._connections == 0 assert len(pool._idle_cache) == 0 # while the one connection is used exclusively, nothing else is served db = pool.connection(False) assert pool._connections == 1 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 with pytest.raises(TooManyConnectionsError): pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() assert db del db assert pool._connections == 0 assert len(pool._idle_cache) == 1 # when it is used as a shared connection, it can serve any number # of shareable requests, but still no dedicated one cache = [pool.connection()] assert pool._connections == 1 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 1 cache.append(pool.connection()) assert pool._connections == 1 assert len(pool._shared_cache) == 1 assert pool._shared_cache[0].shared == 2 else: with pytest.raises(TooManyConnectionsError): pool.connection() with pytest.raises(TooManyConnectionsError): pool.connection(False) if shareable: cache.append(pool.connection(True)) assert pool._connections == 1 assert len(pool._shared_cache) == 1 assert pool._shared_cache[0].shared == 3 else: with pytest.raises(TooManyConnectionsError): pool.connection(True) del cache assert pool._connections == 0 assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 # afterwards the connection can be used exclusively again db = pool.connection(False) assert pool._connections == 1 assert len(pool._idle_cache) == 0 assert db del db assert pool._connections == 0 assert len(pool._idle_cache) == 1 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_raised_to_maxcached(dbapi, threadsafety): """Check that maxconnections is raised to the value of maxcached.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 2, 2, 1) assert pool._maxconnections == 2 assert pool._connections == 0 assert len(pool._idle_cache) == 1 cache = [pool.connection(False)] assert pool._connections == 1 assert len(pool._idle_cache) == 0 cache.append(pool.connection(False)) assert pool._connections == 2 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 with pytest.raises(TooManyConnectionsError): pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_raised_to_mincached(dbapi, threadsafety): """Check that maxconnections is raised to the value of mincached.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, 4, 3, 2, 1, False) assert pool._maxconnections == 4 assert pool._connections == 0 assert len(pool._idle_cache) == 4 cache = [] for _i in range(4): cache.append(pool.connection(False)) assert pool._connections == 4 assert len(pool._idle_cache) == 0 with pytest.raises(TooManyConnectionsError): pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_above_maxshared(dbapi, threadsafety): """Check that shared connections leave room for dedicated ones.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 2, 3, 4, False) assert pool._maxconnections == 4 assert pool._connections == 0 assert len(pool._idle_cache) == 1 # four shareable requests only need maxshared=3 connections cache = [] for _i in range(4): cache.append(pool.connection()) assert len(pool._idle_cache) == 0 if shareable: assert pool._connections == 3 assert len(pool._shared_cache) == 3 cache.append(pool.connection()) assert pool._connections == 3 # so the fourth connection is still available as a dedicated one cache.append(pool.connection(False)) assert pool._connections == 4 else: assert pool._connections == 4 with pytest.raises(TooManyConnectionsError): pool.connection() with pytest.raises(TooManyConnectionsError): pool.connection(False) @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_equal_to_maxshared(dbapi, threadsafety): """Check that all connections may be shared when maxshared allows it.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 0, 3, 3, False) assert pool._maxconnections == 3 assert pool._connections == 0 # used exclusively, the three connections are exhausted right away cache = [] for _i in range(3): cache.append(pool.connection(False)) assert pool._connections == 3 with pytest.raises(TooManyConnectionsError): pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection(True) # used as shared connections, they can serve any number of requests cache = [] assert pool._connections == 0 for _i in range(3): cache.append(pool.connection()) assert pool._connections == 3 if shareable: for _i in range(3): cache.append(pool.connection()) assert pool._connections == 3 else: with pytest.raises(TooManyConnectionsError): pool.connection() with pytest.raises(TooManyConnectionsError): pool.connection(False) @pytest.mark.parametrize("threadsafety", [1, 2]) @pytest.mark.parametrize("maxconnections", [0, None]) def test_maxconnections_unlimited(dbapi, threadsafety, maxconnections): """Check that the number of connections is unlimited by default.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 0, 3, maxconnections) assert pool._maxconnections == 0 assert pool._connections == 0 cache = [] for _i in range(10): cache.append(pool.connection(False)) cache.append(pool.connection()) assert cache if shareable: # the ten shareable requests need only maxshared=3 connections assert pool._connections == 13 assert len(pool._shared_cache) == 3 else: assert pool._connections == 20 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_maxconnections_blocking(dbapi, threadsafety): """Check that a thread waits for a connection when blocking is set.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 1, 1, 1, 1, True) assert pool._maxconnections == 1 assert pool._connections == 0 assert len(pool._idle_cache) == 1 db = pool.connection(False) assert pool._connections == 1 assert len(pool._idle_cache) == 0 def connection(): db = pool.connection() cursor = db.cursor() cursor.execute('set thread') cursor.close() db.close() # the thread cannot get a connection and blocks instead of failing thread = Thread(target=connection) thread.start() thread.join(0.1) assert thread.is_alive() assert pool._connections == 1 assert len(pool._idle_cache) == 0 if shareable: assert len(pool._shared_cache) == 0 session = db._con._con.session assert session == ['rollback'] # releasing the connection unblocks the thread, which then runs to the end del db thread.join(0.1) assert not thread.is_alive() assert pool._connections == 0 assert len(pool._idle_cache) == 1 if shareable: assert len(pool._shared_cache) == 0 db = pool.connection(False) assert pool._connections == 1 assert len(pool._idle_cache) == 0 assert session == ['rollback', 'rollback', 'thread', 'rollback'] assert db del db @pytest.mark.parametrize("threadsafety", [1, 2]) @pytest.mark.parametrize("maxusage", [0, 3, 7]) def test_maxusage(dbapi, threadsafety, maxusage): """Check that connections are reset when used too often.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, 0, 0, 0, 1, False, maxusage) assert pool._maxusage == maxusage assert len(pool._idle_cache) == 0 db = pool.connection(False) assert db._con._maxusage == maxusage assert len(pool._idle_cache) == 0 assert db._con._con.open_cursors == 0 assert db._usage == 0 assert db._con._con.num_uses == 0 assert db._con._con.num_queries == 0 for i in range(20): cursor = db.cursor() assert db._con._con.open_cursors == 1 cursor.execute(f'select test{i}') r = cursor.fetchone() assert r == f'test{i}' cursor.close() assert db._con._con.open_cursors == 0 j = i % maxusage + 1 if maxusage else i + 1 assert db._usage == j assert db._con._con.num_uses == j assert db._con._con.num_queries == j db.cursor().callproc('test') assert db._con._con.open_cursors == 0 assert db._usage == j + 1 assert db._con._con.num_uses == j + 1 assert db._con._con.num_queries == j @pytest.mark.parametrize("threadsafety", [1, 2]) def test_setsession(dbapi, threadsafety): """Check that the session is prepared after every reopening.""" dbapi.threadsafety = threadsafety setsession = ('set time zone', 'set datestyle') pool = PooledDB(dbapi, 0, 0, 0, 1, False, None, setsession) assert pool._setsession == setsession db = pool.connection(False) assert db._setsession_sql == setsession assert db._con._con.session == ['time zone', 'datestyle'] db.cursor().execute('select test') db.cursor().execute('set test1') assert db._usage == 2 assert db._con._con.num_uses == 4 assert db._con._con.num_queries == 1 assert db._con._con.session == ['time zone', 'datestyle', 'test1'] db.close() db = pool.connection(False) assert db._setsession_sql == setsession assert db._con._con.session == \ ['time zone', 'datestyle', 'test1', 'rollback'] db._con._con.close() db.cursor().execute('select test') db.cursor().execute('set test2') assert db._con._con.session == ['time zone', 'datestyle', 'test2'] @pytest.mark.parametrize("threadsafety", [1, 2]) @pytest.mark.parametrize("pool_args", [ (2,), # two idle connections are cached up front (0, 0, 2), # two connections may be shared ]) def test_one_thread_two_connections(dbapi, threadsafety, pool_args): """Check that one thread can use two connections at once.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, *pool_args) # both connections are distinct and count their queries separately db1 = pool.connection() for _i in range(5): db1.cursor().execute('select test') db2 = pool.connection() assert db1 != db2 assert db1._con != db2._con for _i in range(7): db2.cursor().execute('select test') assert db1._con._con.num_queries == 5 assert db2._con._con.num_queries == 7 # releasing the first one and taking it again keeps them distinct del db1 db1 = pool.connection() assert db1 != db2 assert db1._con != db2._con for _i in range(3): db1.cursor().execute('select test') assert db1._con._con.num_queries == 8 db2.cursor().execute('select test') assert db2._con._con.num_queries == 8 @pytest.mark.parametrize("threadsafety", [1, 2]) def test_one_thread_two_shared_connections(dbapi, threadsafety): """Check that one thread can use the same connection twice.""" dbapi.threadsafety = threadsafety shareable = threadsafety > 1 pool = PooledDB(dbapi, 0, 0, 1) # only one connection may be shared, so both requests get the same one db1 = pool.connection() db2 = pool.connection() assert db1 != db2 if shareable: assert db1._con == db2._con else: assert db1._con != db2._con # but a dedicated connection is always a separate one del db1 db1 = pool.connection(False) assert db1 != db2 assert db1._con != db2._con @pytest.mark.parametrize("threadsafety", [1, 2]) def test_three_threads_two_connections(dbapi, threadsafety): """Check that a third thread must wait for a free connection.""" dbapi.threadsafety = threadsafety pool = PooledDB(dbapi, 2, 2, 0, 2, True) queue = Queue(3) def connection(): queue.put(pool.connection(), timeout=1) for _i in range(3): Thread(target=connection).start() db1 = queue.get(timeout=1) db2 = queue.get(timeout=1) assert db1 != db2 db1_con = db1._con db2_con = db2._con assert db1_con != db2_con with pytest.raises(Empty): queue.get(timeout=0.1) del db1 db1 = queue.get(timeout=1) assert db1 != db2 assert db1._con != db2._con assert db1._con == db1_con pool = PooledDB(dbapi, 2, 2, 1, 2, True) db1 = pool.connection(False) db2 = pool.connection(False) assert db1 != db2 db1_con = db1._con db2_con = db2._con assert db1_con != db2_con Thread(target=connection).start() with pytest.raises(Empty): queue.get(timeout=0.1) del db1 db1 = queue.get(timeout=1) assert db1 != db2 assert db1._con != db2._con assert db1._con == db1_con def test_ping_check_never_dedicated(dbapi, ping_con_cls): """Check that dedicated connections are not pinged when ping is 0.""" pool = PooledDB(dbapi, 1, 1, 0, 0, False, None, None, True, None, 0) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 0 # the broken connection is returned to the pool and handed out again # without being pinged, so the breakage goes unnoticed db._con.close() db.close() db = pool.connection() assert not db._con._con.valid assert ping_con_cls.num_pings == 0 def test_ping_check_never_shared(dbapi, ping_con_cls): """Check that shared connections are not pinged when ping is 0.""" pool = PooledDB(dbapi, 1, 1, 1, 0, False, None, None, True, None, 0) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 0 db._con.close() db = pool.connection() assert not db._con._con.valid assert ping_con_cls.num_pings == 0 def test_ping_check_when_fetched_dedicated(dbapi, ping_con_cls): """Check that dedicated connections are pinged when ping is 1.""" pool = PooledDB(dbapi, 1, 1, 0, 0, False, None, None, True, None, 1) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 1 # the broken connection is pinged when fetched from the pool again, # so it is transparently reopened db._con.close() db.close() db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 2 def test_ping_check_when_fetched_shared(dbapi, ping_con_cls): """Check that shared connections are pinged when ping is 1.""" pool = PooledDB(dbapi, 1, 1, 1, 0, False, None, None, True, None, 1) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 1 db._con.close() db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 2 def test_ping_check_with_sql_when_fetched(dbapi, con_cls): """Check that connections can be checked with an SQL statement.""" pool = PooledDB( dbapi, 1, 1, 0, 0, False, None, None, True, None, 'select 1') db = pool.connection() assert db._con._con.valid # the statement has been executed instead of using the ping method assert db._con._con.num_queries == 1 assert con_cls.num_pings == 0 # the broken connection is checked when fetched from the pool again, # so it is transparently reopened db._con.close() db.close() db = pool.connection() assert db._con._con.valid assert con_cls.num_pings == 0 def test_ping_check_when_cursor_created(dbapi, ping_con_cls): """Check that connections are pinged when ping is 2.""" pool = PooledDB(dbapi, 1, 1, 1, 0, False, None, None, True, None, 2) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 0 # fetching the broken connection does not ping it yet db._con.close() db = pool.connection() assert not db._con._con.valid assert ping_con_cls.num_pings == 0 # but creating a cursor does, which reopens the connection db.cursor() assert db._con._con.valid assert ping_con_cls.num_pings == 1 def test_ping_check_when_query_executed(dbapi, ping_con_cls): """Check that connections are pinged when ping is 4.""" pool = PooledDB(dbapi, 1, 1, 1, 0, False, None, None, True, None, 4) db = pool.connection() assert db._con._con.valid assert ping_con_cls.num_pings == 0 # neither fetching the broken connection nor creating a cursor pings it db._con.close() db = pool.connection() assert not db._con._con.valid assert ping_con_cls.num_pings == 0 cursor = db.cursor() db._con.close() assert not db._con._con.valid assert ping_con_cls.num_pings == 0 # only executing a query does, which reopens the connection cursor.execute('select test') assert db._con._con.valid assert ping_con_cls.num_pings == 1 @pytest.mark.parametrize("pool_args", [ (0, 1, 1), # one connection that may be shared (1, 1, 0), # one cached connection that is always dedicated ]) def test_failed_transaction(dbapi, pool_args): """Check that a failed transaction is reported and recovered from.""" pool = PooledDB(dbapi, *pool_args) db = pool.connection() cursor = db.cursor() # outside of a transaction, a broken connection is silently reopened db._con._con.close() cursor.execute('select test') # inside a transaction, the failure must be reported instead db.begin() db._con._con.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test') # but the connection is usable again afterwards cursor.execute('select test') # cancelling the transaction restores the silent reopening as well db.begin() db.cancel() db._con._con.close() cursor.execute('select test') def test_shared_in_transaction(dbapi): """Check that a connection in a transaction is not shared.""" pool = PooledDB(dbapi, 0, 1, 1) db = pool.connection() db.begin() # the only connection is in a transaction now, so it cannot be shared # and no other connection may be created either pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() def test_shared_in_transaction_blocking(dbapi): """Check that a thread waits for a shared connection in a transaction.""" pool = PooledDB(dbapi, 0, 0, 1, 0, True) db = pool.connection() con = db._con db.begin() # the only connection that may be shared is in a transaction now shared = [] def connection(): shared.append(pool.connection()) thread = Thread(target=connection) thread.start() thread.join(0.1) # the thread cannot share that connection and blocks instead of failing assert thread.is_alive() assert not shared db.commit() # the thread is woken up when a connection is put back into the pool, # and then finds the connection shareable again pool.dedicated_connection().close() thread.join(0.1) assert not thread.is_alive() assert len(shared) == 1 assert shared[0]._con is con def test_shared_in_transaction_becoming_idle(dbapi): """Check that a thread waiting for a shared connection survives unshare.""" pool = PooledDB(dbapi, 0, 0, 1, 0, True) db = pool.connection() con = db._con db.begin() # the only connection that may be shared is in a transaction now shared = [] errors = [] def connection(): try: shared.append(pool.connection()) except Exception as error: errors.append(error) thread = Thread(target=connection) thread.start() thread.join(0.1) # the thread cannot share that connection and blocks instead of failing assert thread.is_alive() assert not shared # closing the connection takes it out of the shared cache completely, # so the waiting thread is woken up to find an empty shared cache; # it must then get the connection from the idle cache instead of # trying to share a connection that is no longer there db.close() thread.join(0.1) assert not thread.is_alive() assert not errors assert len(shared) == 1 assert shared[0]._con is con def test_shared_in_transaction_with_two_connections(dbapi): """Check that sharing prefers connections without a transaction.""" pool = PooledDB(dbapi, 0, 2, 2) # the first two requests use the two connections that may be shared db1 = pool.connection() db2 = pool.connection() assert db2._con is not db1._con db2.close() db2 = pool.connection() assert db2._con is not db1._con # the third request shares the least used one db = pool.connection() assert db._con is db1._con db.close() # when that one is in a transaction, the other one is shared instead db1.begin() db = pool.connection() assert db._con is db2._con db.close() # when both are in a transaction, nothing can be shared any more db2.begin() pool.connection(False) with pytest.raises(TooManyConnectionsError): pool.connection() # ending one of the transactions makes that connection shareable again db1.rollback() db = pool.connection() assert db._con is db1._con def test_reset_transaction(dbapi): """Check how connections are reset when returned to the pool.""" pool = PooledDB(dbapi, 1, 1, 0) db = pool.connection() db.begin() con = db._con assert con._transaction assert con._con.session == ['rollback'] db.close() assert pool.connection()._con is con assert not con._transaction assert con._con.session == ['rollback'] * 3 pool = PooledDB(dbapi, 1, 1, 0, reset=False) db = pool.connection() db.begin() con = db._con assert con._transaction assert con._con.session == [] db.close() assert pool.connection()._con is con assert not con._transaction assert con._con.session == ['rollback'] def test_context_manager(dbapi): """Check that connection and cursor can be used as context managers.""" pool = PooledDB(dbapi, 1, 1, 1) con = pool._idle_cache[0]._con with pool.connection() as db: assert hasattr(db, '_shared_con') assert not pool._idle_cache assert con.valid with db.cursor() as cursor: assert con.open_cursors == 1 cursor.execute('select test') r = cursor.fetchone() assert con.open_cursors == 0 assert r == 'test' assert con.num_queries == 1 assert pool._idle_cache with pool.dedicated_connection() as db: assert not hasattr(db, '_shared_con') assert not pool._idle_cache with db.cursor() as cursor: assert con.open_cursors == 1 cursor.execute('select test') r = cursor.fetchone() assert con.open_cursors == 0 assert r == 'test' assert con.num_queries == 2 assert pool._idle_cache def test_shared_db_connection_create(dbapi): """Check that a shared connection starts with one share.""" db_con = dbapi.connect() con = SharedDBConnection(db_con) assert con.con == db_con assert con.shared == 1 def test_shared_db_connection_share_and_unshare(dbapi): """Check that the number of shares is counted correctly.""" con = SharedDBConnection(dbapi.connect()) assert con.shared == 1 con.share() assert con.shared == 2 con.share() assert con.shared == 3 con.unshare() assert con.shared == 2 con.unshare() assert con.shared == 1 def test_shared_db_connection_compare(dbapi): """Check that shared connections are ordered by transaction and shares.""" con1 = SharedDBConnection(dbapi.connect()) con1.con._transaction = False con2 = SharedDBConnection(dbapi.connect()) con2.con._transaction = False # connections with the same number of shares are ranked equally assert not con1 < con2 assert not con2 < con1 # the connection with fewer shares is preferred con2.share() assert con1 < con2 assert not con2 < con1 assert con1 <= con2 assert not con1 >= con2 # but connections in a transaction always come last con1.con._transaction = True assert not con1 < con2 assert con2 < con1 assert con1 > con2 assert not con1 <= con2 def test_shared_db_connection_equality(dbapi): """Check that shared connections are only equal to themselves.""" con1 = SharedDBConnection(dbapi.connect()) con1.con._transaction = False con2 = SharedDBConnection(dbapi.connect()) con2.con._transaction = False same_as_con1 = con1 # the ordering only serves to pick the least shared connection, # it does not make equally ranked connections interchangeable assert con1 == same_as_con1 assert con1 != con2 # equal connections must therefore also have equal hash values # (this used to be violated, since the equality was based on the # values while the hash value was based on the identity) assert hash(con1) == hash(same_as_con1) assert len({con1, con2, same_as_con1}) == 2 # and removing one connection from a list of connections # must not remove another connection that is ranked the same cache = [con1, con2] cache.remove(con2) assert len(cache) == 1 assert cache[0] is con1 def test_shared_db_connection_hash(dbapi): """Check that shared connections stay hashable.""" # defining __eq__ would otherwise make the class unhashable con = SharedDBConnection(dbapi.connect()) hashed = hash(con) assert hash(con) == hashed # the hash is based on the identity, so it does not change # when the number of shares of the connection changes con.share() assert hash(con) == hashed con.unshare() assert hash(con) == hashed def timeout_is_not_fatal(error): """Treat a deliberate server side timeout as not fatal.""" return not error.args or error.args[0] != 3024 def test_isfatal_default(dbapi): """Check that no error check is configured by default.""" pool = PooledDB(dbapi, 1) assert pool._isfatal is None db = pool.connection() assert db._con._isfatal is None def test_isfatal(dbapi): """Check that isfatal is passed on and can veto the failover.""" pool = PooledDB(dbapi, 1, isfatal=timeout_is_not_fatal) assert pool._isfatal is timeout_is_not_fatal db = pool.connection() assert db._con._isfatal is timeout_is_not_fatal dbapi.Connection.num_timeouts = 0 cursor = db.cursor() with pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 # the failover mechanism still applies to other failures db._con._con.close() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_no_failover(dbapi): """Check that no_failover() suspends the failover mechanism.""" pool = PooledDB(dbapi, 1) dbapi.Connection.num_timeouts = 0 with pool.connection() as db, db.cursor() as cursor: with cursor.no_failover(), pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 cursor.execute('select test') # connection is still usable assert cursor.fetchone() == 'test' def test_dbapi_connection(dbapi): """Check that the underlying DB-API 2 objects are accessible.""" pool = PooledDB(dbapi, 1) db = pool.connection() con = db.dbapi_connection assert isinstance(con, dbapi.Connection) assert con is db._con._con cursor = db.cursor() assert cursor.dbapi_cursor is cursor._cursor WebwareForPython-DBUtils-8de754c/tests/test_pooled_pg.py000066400000000000000000000362411524214146500234450ustar00rootroot00000000000000"""Test the PooledPg module. Note: We don't test performance here, so the test does not predicate whether PooledPg actually will help in improving performance or not. We also assume that the underlying SteadyPg connections are tested. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pg import pytest from dbutils.pooled_pg import ( InvalidConnectionError, PooledPg, TooManyConnectionsError, ) from dbutils.steady_pg import SteadyPgConnection def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, pooled_pg assert pooled_pg.__version__ == __version__ assert PooledPg.version == __version__ def test_create_connection(): """Check that the pool creates connections with the given parameters.""" pool = PooledPg( 1, 1, 0, False, None, None, False, 'PooledPgTestDB', user='PooledPgTestUser') assert hasattr(pool, '_cache') assert pool._cache.qsize() == 1 assert hasattr(pool, '_maxusage') assert pool._maxusage is None assert hasattr(pool, '_setsession') assert pool._setsession is None assert hasattr(pool, '_reset') assert not pool._reset # mincached=1, so one steady connection has been created up front db_con = pool._cache.get(0) pool._cache.put(db_con, 0) assert isinstance(db_con, SteadyPgConnection) # taking it out of the pool empties the cache db = pool.connection() assert pool._cache.qsize() == 0 assert hasattr(db, '_con') assert db._con == db_con assert hasattr(db, 'query') assert hasattr(db, 'num_queries') assert db.num_queries == 0 assert hasattr(db, '_maxusage') assert db._maxusage == 0 assert hasattr(db, '_setsession_sql') assert db._setsession_sql is None # the connect parameters have been passed down to the connection assert hasattr(db, 'dbname') assert db.dbname == 'PooledPgTestDB' assert hasattr(db, 'user') assert db.user == 'PooledPgTestUser' db.query('select test') assert db.num_queries == 1 def test_create_connection_without_parameters(): """Check that a pool can be created without connect parameters.""" pool = PooledPg(1) db = pool.connection() assert hasattr(db, 'dbname') assert db.dbname is None assert hasattr(db, 'user') assert db.user is None assert hasattr(db, 'num_queries') assert db.num_queries == 0 def test_create_connection_with_settings(): """Check that maxusage and setsession are passed on to the connections.""" pool = PooledPg(0, 0, 0, False, 3, ('set datestyle',)) assert pool._maxusage == 3 assert pool._setsession == ('set datestyle',) db = pool.connection() assert db._maxusage == 3 assert db._setsession_sql == ('set datestyle',) def test_close_connection(): """Check that a closed connection is returned to the pool.""" pool = PooledPg( 0, 1, 0, False, None, None, False, 'PooledPgTestDB', user='PooledPgTestUser') db = pool.connection() assert hasattr(db, '_con') db_con = db._con assert isinstance(db_con, SteadyPgConnection) assert hasattr(pool, '_cache') assert pool._cache.qsize() == 0 assert db.num_queries == 0 db.query('select test') assert db.num_queries == 1 db.close() with pytest.raises(InvalidConnectionError): assert db.num_queries db = pool.connection() assert hasattr(db, 'dbname') assert db.dbname == 'PooledPgTestDB' assert hasattr(db, 'user') assert db.user == 'PooledPgTestUser' assert db.num_queries == 1 db.query('select test') assert db.num_queries == 2 db = pool.connection() assert pool._cache.qsize() == 1 assert pool._cache.get(0) == db_con assert db del db def test_close_all(): """Check that closing the pool keeps the connections accounted for.""" pool = PooledPg(0, 1, 1, False, None, None, False, 'PooledPgTestDB') assert pool._cache.qsize() == 0 assert pool._connections._value == 1 db = pool.connection() db_con = db._con db.close() # the cached connection has already given back its share of the # generally allowed connections when it was returned to the pool assert pool._cache.qsize() == 1 assert pool._connections._value == 1 pool.close() # closing the pool discards the cached connection, # but it must not give back its share a second time assert pool._cache.qsize() == 0 assert pool._connections._value == 1 assert db_con._closed # so the pool still hands out only one connection at a time db = pool.connection() assert db._con is not db_con assert pool._connections._value == 0 with pytest.raises(TooManyConnectionsError): pool.connection() # closing the pool again (this also happens when it is deleted) # must not change the accounting while a connection is still in use pool.close() assert pool._connections._value == 0 # the connection in use gives back its share when it is returned db.close() assert pool._cache.qsize() == 1 assert pool._connections._value == 1 def test_connection_error_does_not_use_up_a_connection(): """Check that a failing connection attempt is properly accounted for.""" pool = PooledPg(0, 0, 1, False, None, None, False, dbname='ok') assert pool._cache.qsize() == 0 assert pool._connections._value == 1 # the mock database raises an error when the database is named 'error' pool._kwargs['dbname'] = 'error' with pytest.raises(pg.InternalError): pool.connection() pool._kwargs['dbname'] = 'ok' # the failed attempt must not have used up the allowed connection assert pool._connections._value == 1 db = pool.connection() assert db.dbname == 'ok' assert pool._connections._value == 0 # but the connection that could be established still counts with pytest.raises(TooManyConnectionsError): pool.connection() db.close() assert pool._connections._value == 1 @pytest.mark.parametrize(("mincached", "maxcached", "cached", "rounds"), [ # every round takes the given number of connections out of the pool # and expects the given number of cached connections after closing them (3, 0, 3, [(3, 3), (6, 6)]), # unlimited cache grows with the demand (3, None, 3, [(3, 3), (6, 6)]), # None is the same as zero here (None, None, 0, [(3, 3), (6, 6)]), # and also when nothing is cached (3, 4, 3, [(3, 3), (6, 4)]), # cache fills up to maxcached (3, 2, 3, [(4, 3)]), # mincached wins when it exceeds maxcached (2, 5, 2, [(10, 5)]), # cache grows from mincached to maxcached ]) def test_min_max_cached(mincached, maxcached, cached, rounds): """Check that the cache is kept between the given bounds.""" pool = PooledPg(mincached, maxcached) assert hasattr(pool, '_cache') assert pool._cache.qsize() == cached cache = [] for taken, cached_again in rounds: for _i in range(taken): cache.append(pool.connection()) assert pool._cache.qsize() == 0 for _i in range(taken): cache.pop().close() assert pool._cache.qsize() == cached_again def test_max_connections(): """Check that the maximum number of connections is enforced.""" pool = PooledPg(1, 2, 3) assert pool._cache.qsize() == 1 cache = [pool.connection() for _i in range(3)] assert pool._cache.qsize() == 0 with pytest.raises(TooManyConnectionsError): pool.connection() assert cache def test_max_connections_unlimited(): """Check that the number of connections is unlimited by default.""" pool = PooledPg(0, 0, None) assert pool._connections is None cache = [pool.connection() for _i in range(10)] assert pool._cache.qsize() == 0 assert cache def test_max_connections_one(): """Check that a pool can be limited to a single connection.""" pool = PooledPg(0, 1, 1, False) assert pool._blocking == 0 assert pool._cache.qsize() == 0 db = pool.connection() assert pool._cache.qsize() == 0 with pytest.raises(TooManyConnectionsError): pool.connection() assert db del db def test_max_connections_raised_to_maxcached(): """Check that maxconnections is raised to the value of maxcached.""" pool = PooledPg(1, 2, 1) assert pool._cache.qsize() == 1 cache = [pool.connection()] assert pool._cache.qsize() == 0 cache.append(pool.connection()) assert pool._cache.qsize() == 0 with pytest.raises(TooManyConnectionsError): pool.connection() def test_max_connections_raised_to_mincached(): """Check that maxconnections is raised to the value of mincached.""" pool = PooledPg(3, 2, 1, False) assert pool._cache.qsize() == 3 cache = [pool.connection() for _i in range(3)] assert len(cache) == 3 assert pool._cache.qsize() == 0 with pytest.raises(TooManyConnectionsError): pool.connection() def test_max_connections_blocking(): """Check that a thread waits for a connection when blocking is set.""" pool = PooledPg(1, 1, 1, True) assert pool._blocking == 1 assert pool._cache.qsize() == 1 db = pool.connection() assert pool._cache.qsize() == 0 def connection(): pool.connection().query('set thread') # the thread cannot get a connection and blocks instead of failing thread = Thread(target=connection) thread.start() thread.join(0.1) assert thread.is_alive() assert pool._cache.qsize() == 0 session = db._con.session assert session == [] # releasing the connection unblocks the thread, which then runs to the end del db thread.join(0.1) assert not thread.is_alive() assert pool._cache.qsize() == 1 db = pool.connection() assert pool._cache.qsize() == 0 assert session == ['thread'] assert db del db def test_one_thread_two_connections(): """Check that one thread can use two connections at once.""" pool = PooledPg(2) db1 = pool.connection() for _i in range(5): db1.query('select test') db2 = pool.connection() assert db1 != db2 assert db1._con != db2._con for _i in range(7): db2.query('select test') assert db1.num_queries == 5 assert db2.num_queries == 7 del db1 db1 = pool.connection() assert db1 != db2 assert db1._con != db2._con assert hasattr(db1, 'query') for _i in range(3): db1.query('select test') assert db1.num_queries == 8 db2.query('select test') assert db2.num_queries == 8 def test_three_threads_two_connections(): """Check that a third thread must wait for a free connection.""" pool = PooledPg(2, 2, 2, True) queue = Queue(3) def connection(): queue.put(pool.connection(), timeout=1) for _i in range(3): Thread(target=connection).start() db1 = queue.get(timeout=1) db2 = queue.get(timeout=1) db1_con = db1._con db2_con = db2._con assert db1 != db2 assert db1_con != db2_con with pytest.raises(Empty): queue.get(timeout=0.1) del db1 db1 = queue.get(timeout=1) assert db1 != db2 assert db1._con != db2._con assert db1._con == db1_con def test_reset_transaction(): """Check that an open transaction is rolled back by default.""" pool = PooledPg(1) db = pool.connection() db.begin() con = db._con assert con._transaction db.query('select test') assert con.num_queries == 1 # the transaction is rolled back, but nothing else is reset db.close() assert pool.connection()._con is con assert not con._transaction assert con.session == ['begin', 'rollback'] assert con.num_queries == 1 def test_reset_transaction_always(): """Check that a rollback is always issued when reset is 1.""" pool = PooledPg(1, reset=1) db = pool.connection() db.begin() con = db._con assert con._transaction # the connection has already been rolled back when it was created assert con.session == ['rollback', 'begin'] db.query('select test') assert con.num_queries == 1 # and it is rolled back again when it is returned to the pool db.close() assert pool.connection()._con is con assert not con._transaction assert con.session == ['rollback', 'begin', 'rollback', 'rollback'] assert con.num_queries == 1 def test_reset_transaction_completely(): """Check that the connection is reopened when reset is 2.""" pool = PooledPg(1, reset=2) db = pool.connection() db.begin() con = db._con assert con._transaction assert con.session == ['begin'] db.query('select test') assert con.num_queries == 1 # the connection is closed and reopened, so session and counters are gone db.close() assert pool.connection()._con is con assert not con._transaction assert con.session == [] assert con.num_queries == 0 def test_reopen_connection(): """Check that a pooled connection in use can be reopened.""" pool = PooledPg(1) db = pool.connection() con = db._con db.query('select test') assert con.num_queries == 1 # the underlying connection is reopened, which resets the counter db.reopen() assert db._con is con assert con.num_queries == 0 assert db.query('select test') == 'test' assert con.num_queries == 1 def test_reopen_connection_after_closing(): """Check that a connection that is back in the pool can be reopened.""" pool = PooledPg(1) db = pool.connection() con = db._con db.close() assert db._con is None # the connection cannot be used any more when it has been closed with pytest.raises(InvalidConnectionError): db.query # noqa: B018 # but reopening takes another connection out of the pool db.reopen() assert db._con is not None assert db.query('select test') == 'test' assert con.num_queries == 1 def test_reopen_connection_does_not_nest_the_proxy(): """Check that reopening does not wrap another pooled connection.""" pool = PooledPg(1, 2, 3) db = pool.connection() con = db._con db.close() db.reopen() # the proxy must wrap the steady connection itself, not another proxy, # since otherwise the pool cache would fill up with nested proxies assert db._con is con assert isinstance(db._con, SteadyPgConnection) db.close() cached = pool._cache.get_nowait() assert cached is con assert isinstance(cached, SteadyPgConnection) pool._cache.put_nowait(cached) # reopening must not disturb the accounting of the allowed connections dbs = [pool.connection() for _ in range(3)] with pytest.raises(TooManyConnectionsError): pool.connection() while dbs: dbs.pop().close() def test_context_manager(): """Check that the connection can be used as a context manager.""" pool = PooledPg(1, 1, 1) with pool.connection() as db: db_con = db._con._con db.query('select test') assert db_con.num_queries == 1 with pytest.raises(TooManyConnectionsError): pool.connection() with pool.connection() as db: db_con = db._con._con db.query('select test') assert db_con.num_queries == 2 with pytest.raises(TooManyConnectionsError): pool.connection() WebwareForPython-DBUtils-8de754c/tests/test_simple_pooled_db.py000066400000000000000000000114161524214146500247720ustar00rootroot00000000000000"""Test the SimplePooledDB module. Note: We don't test performance here, so the test does not predicate whether SimplePooledDB actually will help in improving performance or not. We also do not test any real world DB-API 2 module, we just mock the basic connection functionality of an arbitrary module. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pytest from dbutils import simple_pooled_db from . import mock_db as dbapi def my_db_pool(threadsafety, max_connections): """Get simple PooledDB connection.""" dbapi_threadsafety = dbapi.threadsafety dbapi.threadsafety = threadsafety try: return simple_pooled_db.PooledDB( dbapi, max_connections, 'SimplePooledDBTestDB', 'SimplePooledDBTestUser') finally: dbapi.threadsafety = dbapi_threadsafety def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__ assert simple_pooled_db.__version__ == __version__ assert simple_pooled_db.PooledDB.version == __version__ @pytest.mark.parametrize("threadsafety", [None, -1, 0, 4]) def test_no_threadsafety(threadsafety): """Check that an unsupported threadsafety level is rejected.""" with pytest.raises(simple_pooled_db.NotSupportedError): my_db_pool(threadsafety, 1) def test_no_threadsafety_attribute(monkeypatch): """Check that a module hiding its threadsafety is rejected.""" monkeypatch.delattr(dbapi, 'threadsafety') with pytest.raises(simple_pooled_db.NotSupportedError): simple_pooled_db.PooledDB( dbapi, 1, 'SimplePooledDBTestDB', 'SimplePooledDBTestUser') @pytest.mark.parametrize("threadsafety", [1, 2, 3]) def test_create_connection(threadsafety): """Check that the pool creates a usable connection.""" dbpool = my_db_pool(threadsafety, 1) db = dbpool.connection() assert hasattr(db, 'cursor') assert hasattr(db, 'open_cursors') assert db.open_cursors == 0 assert hasattr(db, 'database') assert db.database == 'SimplePooledDBTestDB' assert hasattr(db, 'user') assert db.user == 'SimplePooledDBTestUser' cursor = db.cursor() assert cursor is not None assert db.open_cursors == 1 del cursor @pytest.mark.parametrize("threadsafety", [1, 2, 3]) def test_close_connection(threadsafety): """Check that a closed connection is reused by the pool.""" db_pool = my_db_pool(threadsafety, 1) db = db_pool.connection() assert db.open_cursors == 0 cursor1 = db.cursor() assert cursor1 is not None assert db.open_cursors == 1 db.close() assert not hasattr(db, 'open_cursors') db = db_pool.connection() assert hasattr(db, 'database') assert db.database == 'SimplePooledDBTestDB' assert hasattr(db, 'user') assert db.user == 'SimplePooledDBTestUser' assert db.open_cursors == 1 cursor2 = db.cursor() assert cursor2 is not None assert db.open_cursors == 2 del cursor2 del cursor1 @pytest.mark.parametrize("threadsafety", [1, 2, 3]) def test_two_connections(threadsafety): """Check that two connections from the pool stay independent.""" db_pool = my_db_pool(threadsafety, 2) db1 = db_pool.connection() cursors1 = [db1.cursor() for _i_ in range(5)] db2 = db_pool.connection() assert db1 != db2 cursors2 = [db2.cursor() for _i in range(7)] assert db1.open_cursors == 5 assert db2.open_cursors == 7 db1.close() db1 = db_pool.connection() assert db1 != db2 assert hasattr(db1, 'cursor') for _i in range(3): cursors1.append(db1.cursor()) assert db1.open_cursors == 8 cursors2.append(db2.cursor()) assert db2.open_cursors == 8 del cursors2 del cursors1 def test_threadsafety_1(): """Check that threads get dedicated connections and have to wait.""" db_pool = my_db_pool(1, 2) queue = Queue(3) def connection(): queue.put(db_pool.connection()) threads = [Thread(target=connection).start() for _i in range(3)] assert len(threads) == 3 db1 = queue.get(timeout=1) db2 = queue.get(timeout=1) assert db1 != db2 assert db1._con != db2._con with pytest.raises(Empty): queue.get(timeout=0.1) db2.close() db3 = queue.get(timeout=1) assert db1 != db3 assert db1._con != db3._con @pytest.mark.parametrize("threadsafety", [2, 3]) def test_threadsafety_2(threadsafety): """Check that connections are shared when they are thread-safe.""" dbpool = my_db_pool(threadsafety, 2) db1 = dbpool.connection() db2 = dbpool.connection() cursors = [dbpool.connection().cursor() for _i in range(100)] assert db1.open_cursors == 50 assert db2.open_cursors == 50 assert cursors del cursors WebwareForPython-DBUtils-8de754c/tests/test_simple_pooled_pg.py000066400000000000000000000060571524214146500250200ustar00rootroot00000000000000"""Test the SimplePooledPg module. Note: We don't test performance here, so the test does not predicate whether SimplePooledPg actually will help in improving performance or not. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ from queue import Empty, Queue from threading import Thread import pytest from dbutils import simple_pooled_pg def my_db_pool(max_connections): """Get simple PooledPg connection.""" return simple_pooled_pg.PooledPg( max_connections, 'SimplePooledPgTestDB', 'SimplePooledPgTestUser') def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__ assert simple_pooled_pg.__version__ == __version__ assert simple_pooled_pg.PooledPg.version == __version__ def test_create_connection(): """Check that the pool creates a usable connection.""" db_pool = my_db_pool(1) db = db_pool.connection() assert hasattr(db, 'query') assert hasattr(db, 'num_queries') assert db.num_queries == 0 assert hasattr(db, 'dbname') assert db.dbname == 'SimplePooledPgTestDB' assert hasattr(db, 'user') assert db.user == 'SimplePooledPgTestUser' db.query('select 1') assert db.num_queries == 1 def test_close_connection(): """Check that a closed connection is reused by the pool.""" db_pool = my_db_pool(1) db = db_pool.connection() assert db.num_queries == 0 db.query('select 1') assert db.num_queries == 1 db.close() assert not hasattr(db, 'num_queries') db = db_pool.connection() assert hasattr(db, 'dbname') assert db.dbname == 'SimplePooledPgTestDB' assert hasattr(db, 'user') assert db.user == 'SimplePooledPgTestUser' assert db.num_queries == 1 db.query('select 1') assert db.num_queries == 2 def test_two_connections(): """Check that two connections from the pool stay independent.""" db_pool = my_db_pool(2) db1 = db_pool.connection() for _i in range(5): db1.query('select 1') db2 = db_pool.connection() assert db1 != db2 assert db1._con != db2._con for _i in range(7): db2.query('select 1') assert db1.num_queries == 5 assert db2.num_queries == 7 db1.close() db1 = db_pool.connection() assert db1 != db2 assert db1._con != db2._con assert hasattr(db1, 'query') for _i in range(3): db1.query('select 1') assert db1.num_queries == 8 db2.query('select 1') assert db2.num_queries == 8 def test_threads(): """Check that threads must wait for a connection to be returned.""" db_pool = my_db_pool(2) queue = Queue(3) def connection(): queue.put(db_pool.connection()) threads = [Thread(target=connection).start() for _i in range(3)] assert len(threads) == 3 db1 = queue.get(timeout=1) db2 = queue.get(timeout=1) assert db1 != db2 assert db1._con != db2._con with pytest.raises(Empty): queue.get(timeout=0.1) db2.close() db3 = queue.get(timeout=1) assert db1 != db3 assert db1._con != db3._con WebwareForPython-DBUtils-8de754c/tests/test_steady_db.py000066400000000000000000001552501524214146500234350ustar00rootroot00000000000000"""Test the SteadyDB module. Note: We do not test any real DB-API 2 module, but we just mock the basic DB-API 2 connection functionality. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ import pytest from dbutils.steady_db import ( InvalidCursorError, SteadyDBConnection, SteadyDBCursor, ) from dbutils.steady_db import connect as steady_db_connect from . import mock_db as dbapi def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, steady_db assert steady_db.__version__ == __version__ assert steady_db.SteadyDBConnection.version == __version__ def test_mocked_connection(): """Check that the mocked DB-API 2 connection behaves as expected.""" db = dbapi.connect( 'SteadyDBTestDB', user='SteadyDBTestUser') assert hasattr(db, 'database') assert db.database == 'SteadyDBTestDB' assert hasattr(db, 'user') assert db.user == 'SteadyDBTestUser' assert hasattr(db, 'cursor') assert hasattr(db, 'close') assert hasattr(db, 'open_cursors') assert hasattr(db, 'num_uses') assert hasattr(db, 'num_queries') assert hasattr(db, 'session') assert hasattr(db, 'valid') assert db.valid # cursors are counted while they are open, # no matter whether they are closed explicitly or garbage collected assert db.open_cursors == 0 for _i in range(3): cursor = db.cursor() assert db.open_cursors == 1 cursor.close() assert db.open_cursors == 0 cursor = [] for i in range(3): cursor.append(db.cursor()) assert db.open_cursors == i + 1 del cursor assert db.open_cursors == 0 cursor = db.cursor() assert hasattr(cursor, 'execute') assert hasattr(cursor, 'fetchone') assert hasattr(cursor, 'callproc') assert hasattr(cursor, 'close') assert hasattr(cursor, 'valid') assert cursor.valid assert db.open_cursors == 1 # queries and procedure calls both count as uses, # but only the queries are counted as queries for i in range(3): assert db.num_uses == i assert db.num_queries == i cursor.execute(f'select test{i}') assert cursor.fetchone() == f'test{i}' assert cursor.valid assert db.open_cursors == 1 for _i in range(4): cursor.callproc('test') cursor.close() assert not cursor.valid assert db.open_cursors == 0 assert db.num_uses == 7 assert db.num_queries == 3 # a closed cursor cannot be used or closed again with pytest.raises(dbapi.InternalError): cursor.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test') # closing the connection resets the counters, # and the connection cannot be used or closed again either assert db.valid db.close() assert not db.valid assert db.num_uses == 0 assert db.num_queries == 0 with pytest.raises(dbapi.InternalError): db.close() with pytest.raises(dbapi.InternalError): db.cursor() def test_mocked_connection_ping(con_cls): """Check that the mocked DB-API 2 connection can be pinged.""" db = dbapi.connect( 'SteadyDBTestDB', user='SteadyDBTestUser') # every attempt to ping is counted, even when there is no ping method assert not con_cls.has_ping assert con_cls.num_pings == 0 with pytest.raises(AttributeError): db.ping() assert con_cls.num_pings == 1 con_cls.has_ping = True assert db.ping() is None assert con_cls.num_pings == 2 # pinging a closed connection reports it as broken db.close() with pytest.raises(dbapi.OperationalError): db.ping() assert con_cls.num_pings == 3 def test_broken_connection(): """Check that errors when opening connection or cursor are reported.""" with pytest.raises(TypeError): SteadyDBConnection(None) with pytest.raises(TypeError): SteadyDBCursor(None) db = steady_db_connect(dbapi, database='ok') for _i in range(3): db.close() del db with pytest.raises(dbapi.OperationalError): steady_db_connect(dbapi, database='error') db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() for _i in range(3): cursor.close() cursor = db.cursor('ok') for _i in range(3): cursor.close() with pytest.raises(dbapi.OperationalError): db.cursor('error') @pytest.mark.parametrize("closeable", [False, True]) def test_close(closeable): """Check that closing is only allowed when the connection is closeable.""" db = steady_db_connect(dbapi, closeable=closeable) assert db._con.valid db.close() assert closeable ^ db._con.valid db.close() assert closeable ^ db._con.valid db._close() assert not db._con.valid db._close() assert not db._con.valid def test_connection(): """Check that the connection wraps the DB-API 2 connection.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert isinstance(db, SteadyDBConnection) assert hasattr(db, '_con') assert hasattr(db, '_usage') assert db._usage == 0 assert hasattr(db._con, 'valid') assert db._con.valid assert hasattr(db._con, 'cursor') assert hasattr(db._con, 'close') assert hasattr(db._con, 'open_cursors') assert hasattr(db._con, 'num_uses') assert hasattr(db._con, 'num_queries') assert hasattr(db._con, 'session') assert hasattr(db._con, 'database') assert db._con.database == 'SteadyDBTestDB' assert hasattr(db._con, 'user') assert db._con.user == 'SteadyDBTestUser' assert hasattr(db, 'cursor') assert hasattr(db, 'close') # cursors created through the wrapper are counted like the mocked ones assert db._con.open_cursors == 0 for _i in range(3): cursor = db.cursor() assert db._con.open_cursors == 1 cursor.close() assert db._con.open_cursors == 0 cursor = [] for i in range(3): cursor.append(db.cursor()) assert db._con.open_cursors == i + 1 del cursor assert db._con.open_cursors == 0 cursor = db.cursor() assert hasattr(cursor, 'execute') assert hasattr(cursor, 'fetchone') assert hasattr(cursor, 'callproc') assert hasattr(cursor, 'close') assert hasattr(cursor, 'valid') assert cursor.valid assert db._con.open_cursors == 1 # the wrapper keeps its own usage count next to the mocked counters for i in range(3): assert db._usage == i assert db._con.num_uses == i assert db._con.num_queries == i cursor.execute(f'select test{i}') assert cursor.fetchone() == f'test{i}' assert cursor.valid assert db._con.open_cursors == 1 for _i in range(4): cursor.callproc('test') cursor.close() assert not cursor.valid assert db._con.open_cursors == 0 assert db._usage == 7 assert db._con.num_uses == 7 assert db._con.num_queries == 3 # unlike the mocked cursor, the wrapped one can be closed and used again cursor.close() cursor.execute('select test8') assert cursor.valid assert db._con.open_cursors == 1 assert cursor.fetchone() == 'test8' assert db._usage == 8 assert db._con.num_uses == 8 assert db._con.num_queries == 4 def test_connection_close(): """Check that closing the connection closes the DB-API 2 connection.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') cursor = db.cursor() cursor.execute('select test') assert db._usage == 1 assert db._con.valid # the underlying connection is closed, but the wrapper keeps its # usage count and can be closed again without an error db.close() assert not db._con.valid assert db._con.open_cursors == 0 assert db._usage == 1 assert db._con.num_uses == 0 assert db._con.num_queries == 0 with pytest.raises(dbapi.InternalError): db._con.close() db.close() with pytest.raises(dbapi.InternalError): db._con.cursor() def test_connection_reopen(): """Check that the DB-API 2 connection is transparently reopened.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') db.close() # asking for a cursor reopens the underlying connection cursor = db.cursor() assert db._con.valid cursor.execute('select test11') assert cursor.fetchone() == 'test11' cursor.execute('select test12') assert cursor.fetchone() == 'test12' cursor.callproc('test') assert db._usage == 3 assert db._con.num_uses == 3 assert db._con.num_queries == 2 # a second cursor works on the same reopened connection cursor2 = db.cursor() assert db._con.open_cursors == 2 cursor2.execute('select test13') assert cursor2.fetchone() == 'test13' assert db._con.num_queries == 3 db.close() assert db._con.open_cursors == 0 assert db._con.num_queries == 0 def test_connection_reopen_when_broken(): """Check that a broken connection or cursor is transparently replaced.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') cursor = db.cursor() assert cursor.valid cursor.callproc('test') # invalidating the underlying cursor makes the wrapper get a new one cursor._cursor.valid = False assert not cursor.valid with pytest.raises(dbapi.InternalError): cursor._cursor.callproc('test') cursor.callproc('test') assert cursor.valid cursor._cursor.callproc('test') assert db._usage == 2 assert db._con.num_uses == 3 # invalidating the connection as well makes it reconnect, # which resets the usage count of the wrapper too db._con.valid = cursor._cursor.valid = False cursor.callproc('test') assert cursor.valid assert db._usage == 1 assert db._con.num_uses == 1 # the session is passed on to the underlying connection as usual cursor.execute('set this') db.commit() cursor.execute('set that') db.rollback() assert db._con.session == ['this', 'commit', 'that', 'rollback'] def test_connection_context_handler(): """Check that the context handler commits or rolls back.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert db._con.session == [] with db as con: con.cursor().execute('select test') assert db._con.session == ['commit'] try: with db as con: con.cursor().execute('error') except dbapi.ProgrammingError: error = True else: error = False assert error assert db._con.session == ['commit', 'rollback'] def test_cursor_context_handler(): """Check that the cursor is closed by the context handler.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert db._con.open_cursors == 0 with db.cursor() as cursor: assert db._con.open_cursors == 1 cursor.execute('select test') assert cursor.fetchone() == 'test' assert db._con.open_cursors == 0 def test_cursor_as_iterator_provided(): """Check that an iterator provided by the cursor is used.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert db._con.open_cursors == 0 cursor = db.cursor() assert db._con.open_cursors == 1 cursor.execute('select test') _cursor = cursor._cursor try: assert not hasattr(_cursor, 'iter') _cursor.__iter__ = lambda: ['test-iter'] assert list(iter(cursor)) == ['test'] finally: del _cursor.__iter__ cursor.close() assert db._con.open_cursors == 0 def test_cursor_as_iterator_created(): """Check that an iterator is created when the cursor has none.""" db = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert db._con.open_cursors == 0 cursor = db.cursor() assert db._con.open_cursors == 1 cursor.execute('select test') assert list(iter(cursor)) == ['test'] cursor.close() assert db._con.open_cursors == 0 def test_connection_creator_function(): """Check that a creator function can be used instead of a module.""" db1 = steady_db_connect( dbapi, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') db2 = steady_db_connect( dbapi.connect, 0, None, None, None, True, None, 'SteadyDBTestDB', user='SteadyDBTestUser') assert db1.dbapi() == db2.dbapi() assert db1.threadsafety() == db2.threadsafety() assert db1._creator == db2._creator assert db1._args == db2._args assert db1._kwargs == db2._kwargs db2.close() db1.close() def creator_of_this_module(*args, **kwargs): """Create a mocked connection from inside this test module.""" return dbapi.connect(*args, **kwargs) def test_connection_creator_function_of_other_module(): """Check that the DB-API 2 module is determined via the connection.""" db = steady_db_connect(creator_of_this_module, database='ok') # the module cannot be derived from a creator function defined here, # but it is found via the module of the connection that it returns assert db.dbapi() is dbapi assert db.threadsafety() == dbapi.threadsafety assert db._failures == ( dbapi.OperationalError, dbapi.InterfaceError, dbapi.InternalError) # the failover mechanism works just as with a module as creator db.close() cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_connection_creator_class(): """Check that the connection class itself can be used as creator.""" db = steady_db_connect(dbapi.Connection, database='ok') # the module of the creator provides a connect() function, but it is # a different one, so the module is determined via the connection assert db.dbapi() is dbapi assert db.threadsafety() == dbapi.threadsafety cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' class CreatorWithDbapi: """A creator module that points to the real DB-API 2 module.""" dbapi = dbapi connect = staticmethod(dbapi.connect) def test_connection_creator_with_dbapi(): """Check that a creator can name its DB-API 2 module explicitly.""" db = steady_db_connect(CreatorWithDbapi, database='ok') assert db.dbapi() is dbapi assert db.threadsafety() == dbapi.threadsafety cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' class ConnectionWithErrorsOnly: """A connection that reveals its module only via its error classes.""" OperationalError = dbapi.OperationalError InterfaceError = dbapi.InterfaceError InternalError = dbapi.InternalError def close(self): """Close the connection.""" def creator_of_hidden_module(): """Create a connection that hides the module it belongs to.""" return ConnectionWithErrorsOnly() def test_connection_module_found_via_error_class(): """Check that the DB-API 2 module is found via the error classes.""" db = steady_db_connect(creator_of_hidden_module) # the module of the connection itself does not provide connect(), # but the module of its error classes does assert db.dbapi() is dbapi assert db.threadsafety() == dbapi.threadsafety def test_connection_with_unknown_threadsafety(monkeypatch): """Check that an undeterminable threadsafety is reported as zero.""" monkeypatch.delattr(dbapi, 'threadsafety') db = steady_db_connect(creator_of_this_module, database='ok') assert db.dbapi() is dbapi assert db.threadsafety() == 0 def unknown_error_cls(name): """Create an error class belonging to an unimportable module.""" return type(name, (Exception,), {'__module__': 'unknown_db_module'}) class UnknownConnection: """A connection that cannot be traced back to a DB-API 2 module. Neither this test module nor the module claimed by the error classes provides a connect() function, so there is no way to determine the DB-API 2 module from such a connection. """ threadsafety = 3 OperationalError = unknown_error_cls('OperationalError') InterfaceError = unknown_error_cls('InterfaceError') InternalError = unknown_error_cls('InternalError') def close(self): """Close the connection.""" class NamelessConnection: """A connection that does not tell anything about its module.""" def close(self): """Close the connection.""" def creator_of_unknown_module(): """Create a connection with an undeterminable DB-API 2 module.""" return UnknownConnection() def creator_with_failures(): """Create such a connection, but make the failures known.""" return UnknownConnection() creator_with_failures.OperationalError = ( UnknownConnection.OperationalError) creator_with_failures.InterfaceError = ( UnknownConnection.InterfaceError) creator_with_failures.InternalError = ( UnknownConnection.InternalError) def test_connection_with_unknown_module(): """Check the fallbacks when the DB-API 2 module cannot be determined.""" db = steady_db_connect(creator_of_unknown_module) with pytest.raises(AttributeError): db.dbapi() # the threadsafety and the failures are taken from the connection assert db.threadsafety() == 3 assert db._failures == ( UnknownConnection.OperationalError, UnknownConnection.InterfaceError, UnknownConnection.InternalError) def test_connection_with_failures_of_creator(): """Check that the failures can also be taken from the creator.""" db = steady_db_connect(creator_with_failures) assert db._failures == ( UnknownConnection.OperationalError, UnknownConnection.InterfaceError, UnknownConnection.InternalError) def test_connection_with_undeterminable_failures(): """Check that undeterminable failures are reported as an error.""" with pytest.raises(AttributeError, match='failure exceptions'): steady_db_connect(NamelessConnection) def test_connection_with_undeterminable_threadsafety(): """Check that an undeterminable threadsafety is reported as an error.""" db = steady_db_connect(NamelessConnection, failures=dbapi.InternalError) with pytest.raises(AttributeError, match='threadsafety'): db.threadsafety() def test_connection_maxusage(): """Check that the connection is reset when used too often.""" db = steady_db_connect(dbapi, 10) cursor = db.cursor() # the connection is silently reset after every 10 uses for i in range(100): cursor.execute(f'select test{i}') r = cursor.fetchone() assert r == f'test{i}' assert db._con.valid j = i % 10 + 1 assert db._usage == j assert db._con.num_uses == j assert db._con.num_queries == j assert db._con.open_cursors == 1 # inside a transaction the reset is postponed until it has been ended, # so the usage count keeps growing beyond the maximum until then db.begin() for i in range(100): cursor.callproc('test') assert db._con.valid if i == 49: db.commit() j = i % 10 + 1 if i > 49 else i + 11 assert db._usage == j assert db._con.num_uses == j j = 0 if i > 49 else 10 assert db._con.num_queries == j # a connection that breaks in between is reset early, # which also restarts the counting of the uses for i in range(10): if i == 7: db._con.valid = cursor._cursor.valid = False cursor.execute(f'select test{i}') r = cursor.fetchone() assert r == f'test{i}' j = i % 7 + 1 assert db._usage == j assert db._con.num_uses == j assert db._con.num_queries == j for i in range(10): if i == 5: db._con.valid = cursor._cursor.valid = False cursor.callproc('test') j = (i + (3 if i < 5 else -5)) % 10 + 1 assert db._usage == j assert db._con.num_uses == j j = 3 if i < 5 else 0 assert db._con.num_queries == j # closing the connection restarts the counting as well db.close() cursor.execute('select test1') assert cursor.fetchone() == 'test1' assert db._usage == 1 assert db._con.num_uses == 1 assert db._con.num_queries == 1 def test_connection_setsession_error(): """Check that a session that cannot be prepared is reported.""" with pytest.raises(dbapi.ProgrammingError): steady_db_connect(dbapi, 0, ('set time zone', 'error')) # the connection is closed again when the session cannot be prepared connections = [] def creator(*args, **kwargs): con = dbapi.connect(*args, **kwargs) connections.append(con) return con with pytest.raises(dbapi.ProgrammingError): steady_db_connect(creator, 0, ('error',), failures=dbapi.InternalError) assert len(connections) == 1 assert not connections[0].valid def test_connection_setsession(): """Check that the session is prepared after every reopening.""" db = steady_db_connect(dbapi, 3, ('set time zone', 'set datestyle')) assert hasattr(db, '_usage') assert db._usage == 0 assert hasattr(db._con, 'open_cursors') assert db._con.open_cursors == 0 assert hasattr(db._con, 'num_uses') assert db._con.num_uses == 2 assert hasattr(db._con, 'num_queries') assert db._con.num_queries == 0 assert hasattr(db._con, 'session') assert tuple(db._con.session) == ('time zone', 'datestyle') # maxusage is 3, so the connection is reset after every third query # and the session is prepared again from scratch each time for _i in range(11): db.cursor().execute('select test') assert db._con.open_cursors == 0 assert db._usage == 2 assert db._con.num_uses == 4 assert db._con.num_queries == 2 assert db._con.session == ['time zone', 'datestyle'] # a "set" command is added to the session, but is lost on the next reset db.cursor().execute('set test') assert db._con.open_cursors == 0 assert db._usage == 3 assert db._con.num_uses == 5 assert db._con.num_queries == 2 assert db._con.session == ['time zone', 'datestyle', 'test'] db.cursor().execute('select test') assert db._con.open_cursors == 0 assert db._usage == 1 assert db._con.num_uses == 3 assert db._con.num_queries == 1 assert db._con.session == ['time zone', 'datestyle'] db.cursor().execute('set test') assert db._con.open_cursors == 0 assert db._usage == 2 assert db._con.num_uses == 4 assert db._con.num_queries == 1 assert db._con.session == ['time zone', 'datestyle', 'test'] db.cursor().execute('select test') assert db._con.open_cursors == 0 assert db._usage == 3 assert db._con.num_uses == 5 assert db._con.num_queries == 2 assert db._con.session == ['time zone', 'datestyle', 'test'] # the same happens after the connection has been closed explicitly db.close() db.cursor().execute('set test') assert db._con.open_cursors == 0 assert db._usage == 1 assert db._con.num_uses == 3 assert db._con.num_queries == 0 assert db._con.session == ['time zone', 'datestyle', 'test'] db.close() db.cursor().execute('select test') assert db._con.open_cursors == 0 assert db._usage == 1 assert db._con.num_uses == 3 assert db._con.num_queries == 1 assert db._con.session == ['time zone', 'datestyle'] def test_connection_maxusage_must_be_integer(): """Check that maxusage must be an integer when it is given.""" steady_db_connect(dbapi, None) steady_db_connect(dbapi, 42) with pytest.raises(TypeError): steady_db_connect(dbapi, 'unlimited') with pytest.raises(TypeError): steady_db_connect(dbapi, 42.5) def test_connection_failures_must_be_exceptions(): """Check that failures must be exception classes when they are given.""" steady_db_connect(dbapi, failures=None) steady_db_connect(dbapi, failures=dbapi.InternalError) steady_db_connect(dbapi, failures=(dbapi.InternalError,)) with pytest.raises(TypeError): steady_db_connect(dbapi, failures=int) with pytest.raises(TypeError): steady_db_connect(dbapi, failures='InternalError') with pytest.raises(TypeError): steady_db_connect(dbapi, failures=42) def test_connection_failures(): """Check that only the configured failures trigger the failover.""" db = steady_db_connect(dbapi) db.close() db.cursor() db = steady_db_connect(dbapi, failures=dbapi.InternalError) db.close() db.cursor() db = steady_db_connect(dbapi, failures=dbapi.OperationalError) db.close() with pytest.raises(dbapi.InternalError): db.cursor() db = steady_db_connect(dbapi, failures=( dbapi.OperationalError, dbapi.InterfaceError)) db.close() with pytest.raises(dbapi.InternalError): db.cursor() db = steady_db_connect(dbapi, failures=( dbapi.OperationalError, dbapi.InterfaceError, dbapi.InternalError)) db.close() db.cursor() def test_connection_failure_error(): """Check that errors after a successful failover are passed on.""" db = steady_db_connect(dbapi) cursor = db.cursor() db.close() cursor.execute('select test') cursor = db.cursor() db.close() with pytest.raises(dbapi.ProgrammingError): cursor.execute('error') def test_connection_set_sizes(): """Check that input and output sizes are set and kept when reopening.""" db = steady_db_connect(dbapi) cursor = db.cursor() cursor.execute('get sizes') result = cursor.fetchone() assert result == ([], {}) cursor.setinputsizes([7, 42, 6]) cursor.setoutputsize(9) cursor.setoutputsize(15, 3) cursor.setoutputsize(42, 7) cursor.execute('get sizes') result = cursor.fetchone() assert result == ([7, 42, 6], {None: 9, 3: 15, 7: 42}) cursor.execute('get sizes') result = cursor.fetchone() assert result == ([], {}) cursor.setinputsizes([6, 42, 7]) cursor.setoutputsize(7) cursor.setoutputsize(15, 3) cursor.setoutputsize(42, 9) db.close() cursor.execute('get sizes') result = cursor.fetchone() assert result == ([6, 42, 7], {None: 7, 3: 15, 9: 42}) def test_connection_ping(ping_con_cls): """Check that ping() is passed on to the underlying connection.""" db = steady_db_connect(dbapi) assert ping_con_cls.num_pings == 0 assert db.ping() is None assert ping_con_cls.num_pings == 1 # in contrast to the ping check, this does not swallow the error # and does not reopen the connection when it has been closed con = db._con db.close() with pytest.raises(dbapi.OperationalError): db.ping() assert ping_con_cls.num_pings == 2 assert db._con is con def test_connection_ping_without_ping_method(con_cls): """Check that ping() reports when the connection cannot be pinged.""" db = steady_db_connect(dbapi) assert con_cls.num_pings == 0 with pytest.raises(AttributeError): db.ping() assert con_cls.num_pings == 1 def test_connection_ping_check_without_ping_method(con_cls): """Check that pinging is skipped when there is no ping method.""" db = steady_db_connect(dbapi) db.cursor().execute('select test') assert con_cls.num_pings == 0 db.close() db.cursor().execute('select test') assert con_cls.num_pings == 0 # an explicit check attempts a ping, but cannot report anything assert db._ping_check() is None assert con_cls.num_pings == 1 def test_connection_ping_check_always_without_ping_method(con_cls): """Check that pinging is given up when there is no ping method.""" db = steady_db_connect(dbapi, ping=7) # the first attempt fails with an AttributeError, # after which no further pings are attempted at all db.cursor().execute('select test') assert con_cls.num_pings == 1 db.close() db.cursor().execute('select test') assert con_cls.num_pings == 1 assert db._ping_check() is None assert con_cls.num_pings == 1 def test_connection_ping_check_default(ping_con_cls): """Check that connections are pinged on request by default.""" db = steady_db_connect(dbapi) db.cursor().execute('select test') assert ping_con_cls.num_pings == 0 db.close() db.cursor().execute('select test') assert ping_con_cls.num_pings == 0 assert db._ping_check() assert ping_con_cls.num_pings == 1 @pytest.mark.parametrize("ping", range(8)) def test_connection_ping_check_combinations(ping, ping_con_cls): """Check the three ping triggers in all of their bit combinations.""" on_request = bool(ping & 1) # 1 = when _ping_check() is called on_cursor = bool(ping & 2) # 2 = whenever a cursor is created on_execute = bool(ping & 4) # 4 = when a query is executed db = steady_db_connect(dbapi, ping=ping) assert ping_con_cls.num_pings == 0 # while the connection is open, only the configured triggers ping it expected = 0 cursor = db.cursor() expected += on_cursor assert ping_con_cls.num_pings == expected cursor.execute('select test') expected += on_execute assert ping_con_cls.num_pings == expected assert cursor.fetchone() == 'test' # a closed connection is pinged just as often, since the ping reports # it as broken and the connection is then reopened transparently db.close() cursor = db.cursor() expected += on_cursor assert ping_con_cls.num_pings == expected cursor.execute('select test') expected += on_execute assert ping_con_cls.num_pings == expected assert cursor.fetchone() == 'test' # an explicit check pings only when this has been requested, # and it reopens the connection when it has been closed db.close() assert db._ping_check() is (True if on_request else None) expected += on_request assert ping_con_cls.num_pings == expected assert db._con.valid is on_request def test_connection_ping_check_with_sql(con_cls): """Check that connections can be checked with an SQL statement.""" db = steady_db_connect(dbapi, ping='select 1') con = db._con assert con.num_queries == 0 # the statement is executed instead of using the ping method assert db._ping_check() assert con_cls.num_pings == 0 assert con.num_queries == 1 # the cursor used for the check has been closed again assert con.open_cursors == 0 # and the check does not count towards the usage limit assert db._usage == 0 # a broken connection is detected and transparently reopened db.close() assert not con.valid assert db._ping_check() assert db._con is not con assert db._con.valid assert con_cls.num_pings == 0 def test_connection_ping_check_with_sql_when_query_executed(con_cls): """Check that an SQL statement can also be used at other times.""" db = steady_db_connect(dbapi, ping=(4, 'select 1')) con = db._con cursor = db.cursor() # creating a cursor does not check the connection assert con.num_queries == 0 # but executing a query does, in addition to the query itself cursor.execute('select test') assert con.num_queries == 2 assert cursor.fetchone() == 'test' assert con_cls.num_pings == 0 def test_connection_ping_check_with_failing_sql(con_cls): """Check that a failing check statement is not fatal.""" db = steady_db_connect(dbapi, ping='error') con = db._con # the failing statement makes the connection appear broken, # so that it is unnecessarily, but harmlessly reopened assert db._ping_check() assert db._con is not con assert db._con.valid assert con_cls.num_pings == 0 def test_connection_ping_check_with_function(con_cls, pinger): """Check that connections can be checked with a check function.""" db = steady_db_connect(dbapi, ping=pinger) con = db._con assert not pinger.calls assert db._ping_check() # the function is passed the underlying DB-API 2 connection assert pinger.calls == [con] assert con_cls.num_pings == 0 assert con.num_queries == 0 # a broken connection is detected and transparently reopened db.close() assert db._ping_check() assert pinger.calls == [con, con] assert db._con is not con assert db._con.valid def test_connection_ping_check_with_function_at_default_times(pinger): """Check that a check function is used at the default times.""" db = steady_db_connect(dbapi, ping=pinger) # neither creating a cursor nor executing a query checks the connection db.cursor().execute('select test') assert not pinger.calls # but requesting an explicit check does assert db._ping_check() assert len(pinger.calls) == 1 def test_connection_ping_check_with_broken_function(con_cls): """Check that a broken check function does not disable the check.""" def broken_check(_con): raise AttributeError db = steady_db_connect(dbapi, ping=(7, broken_check)) con = db._con assert db._ping_check() # in contrast to a missing ping method, the check is not given up, # the connection is only considered broken and therefore reopened assert db._ping == 7 assert db._con is not con assert db._con.valid assert con_cls.num_pings == 0 def test_connection_ping_check_with_invalid_parameter(): """Check that an invalid check function or statement is rejected.""" with pytest.raises(TypeError): steady_db_connect(dbapi, ping=(1, 42)) with pytest.raises(TypeError): steady_db_connect(dbapi, ping=(1, 'select 1', 2)) # non-integer values without a check function disable the check db = steady_db_connect(dbapi, ping=None) assert db._ping == 0 assert db._pinger is None @pytest.mark.parametrize("ping", range(8)) def test_connection_ping_check_combinations_with_function( ping, con_cls, pinger): """Check the three ping triggers when a check function is used.""" on_request = bool(ping & 1) # 1 = when _ping_check() is called on_cursor = bool(ping & 2) # 2 = whenever a cursor is created on_execute = bool(ping & 4) # 4 = when a query is executed db = steady_db_connect(dbapi, ping=(ping, pinger)) assert not pinger.calls # while the connection is open, only the configured triggers check it expected = 0 cursor = db.cursor() expected += on_cursor assert len(pinger.calls) == expected cursor.execute('select test') expected += on_execute assert len(pinger.calls) == expected assert cursor.fetchone() == 'test' # a closed connection is checked just as often, since the check reports # it as broken and the connection is then reopened transparently db.close() cursor = db.cursor() expected += on_cursor assert len(pinger.calls) == expected cursor.execute('select test') expected += on_execute assert len(pinger.calls) == expected assert cursor.fetchone() == 'test' # an explicit check is made only when this has been requested, # and it reopens the connection when it has been closed db.close() assert db._ping_check() is (True if on_request else None) expected += on_request assert len(pinger.calls) == expected assert db._con.valid is on_request # the ping method has not been used at all assert con_cls.num_pings == 0 def test_connection_ping_check_always_with_cursor(ping_con_cls): """Check that a kept cursor is pinged everywhere when ping is 7.""" db = steady_db_connect(dbapi, ping=7) assert ping_con_cls.num_pings == 0 db.cursor() assert ping_con_cls.num_pings == 1 db.close() cursor = db.cursor() assert ping_con_cls.num_pings == 2 cursor.execute('select test') assert ping_con_cls.num_pings == 3 db.close() cursor = db.cursor() assert ping_con_cls.num_pings == 4 cursor.execute('select test') assert ping_con_cls.num_pings == 5 # when the connection was closed, the cursor is recreated on execution, # which pings the connection a second time db.close() cursor.execute('select test') assert ping_con_cls.num_pings == 7 def test_begin_transaction(): """Check that transparent reopening is suspended after begin().""" db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() cursor.close() cursor.execute('select test12') assert cursor.fetchone() == 'test12' db.begin() cursor = db.cursor() cursor.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test12') cursor.execute('select test12') assert cursor.fetchone() == 'test12' db.close() db.begin() with pytest.raises(dbapi.InternalError): cursor.execute('select test12') cursor.execute('select test12') assert cursor.fetchone() == 'test12' db.begin() with pytest.raises(dbapi.ProgrammingError): cursor.execute('error') cursor.close() cursor.execute('select test12') assert cursor.fetchone() == 'test12' def test_with_begin_extension(): """Check that begin() is passed on to the underlying connection.""" db = steady_db_connect(dbapi, database='ok') db._con._begin_called_with = None def begin(a, b=None, c=7): db._con._begin_called_with = (a, b, c) db._con.begin = begin db.begin(42, 6) cursor = db.cursor() cursor.execute('select test13') assert cursor.fetchone() == 'test13' assert db._con._begin_called_with == (42, 6, 7) def test_cancel_transaction(): """Check that cancel() ends a transaction started with begin().""" db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() db.begin() cursor.execute('select test14') assert cursor.fetchone() == 'test14' db.cancel() cursor.execute('select test14') assert cursor.fetchone() == 'test14' def test_with_cancel_extension(): """Check that cancel() is passed on to the underlying connection.""" db = steady_db_connect(dbapi, database='ok') db._con._cancel_called = None def cancel(): db._con._cancel_called = 'yes' db._con.cancel = cancel db.begin() cursor = db.cursor() cursor.execute('select test15') assert cursor.fetchone() == 'test15' db.cancel() assert db._con._cancel_called == 'yes' def test_reset_transaction(): """Check that closing rolls back a transaction when not closeable.""" db = steady_db_connect(dbapi, database='ok') db.begin() assert not db._con.session db.close() assert not db._con.session db = steady_db_connect(dbapi, database='ok', closeable=False) db.begin() assert not db._con.session db.close() assert db._con.session == ['rollback'] def test_commit_error(): """Check that the connection is reopened when a commit fails.""" db = steady_db_connect(dbapi, database='ok') db.begin() assert not db._con.session assert db._con.valid db.commit() assert db._con.session == ['commit'] assert db._con.valid db.begin() db._con.valid = False con = db._con with pytest.raises(dbapi.InternalError): db.commit() assert not db._con.session assert db._con.valid assert con is not db._con db.begin() assert not db._con.session assert db._con.valid db.commit() assert db._con.session == ['commit'] assert db._con.valid def test_rollback_error(): """Check that the connection is reopened when a rollback fails.""" db = steady_db_connect(dbapi, database='ok') db.begin() assert not db._con.session assert db._con.valid db.rollback() assert db._con.session == ['rollback'] assert db._con.valid db.begin() db._con.valid = False con = db._con with pytest.raises(dbapi.InternalError): db.rollback() assert not db._con.session assert db._con.valid assert con is not db._con db.begin() assert not db._con.session assert db._con.valid db.rollback() assert db._con.session == ['rollback'] assert db._con.valid def test_commit_error_when_reopening_fails(): """Check that a failed commit is reported when there is no recovery.""" db = steady_db_connect(dbapi, database='ok') db.begin() con = db._con # the database goes away and cannot be reached again db._con.valid = False db._kwargs['database'] = 'error' with pytest.raises(dbapi.InternalError): db.commit() # the connection could not be replaced, but the transaction is over assert db._con is con assert not db._transaction def test_rollback_error_when_reopening_fails(): """Check that a failed rollback is reported when there is no recovery.""" db = steady_db_connect(dbapi, database='ok') db.begin() con = db._con db._con.valid = False db._kwargs['database'] = 'error' with pytest.raises(dbapi.InternalError): db.rollback() assert db._con is con assert not db._transaction def test_ping_check_when_reopening_fails(ping_con_cls): """Check that a failed ping is reported when there is no recovery.""" db = steady_db_connect(dbapi, database='ok') db.close() db._kwargs['database'] = 'error' # the ping reports the connection as broken and it cannot be reopened assert db._ping_check() is False assert ping_con_cls.num_pings == 1 assert not db._con.valid # when the database is back, the connection is reopened as usual db._kwargs['database'] = 'ok' assert db._ping_check() is True assert ping_con_cls.num_pings == 2 assert db._con.valid def test_cursor_when_reopening_fails(): """Check that a cursor cannot be created when there is no recovery.""" db = steady_db_connect(dbapi, database='ok') db.close() db._kwargs['database'] = 'error' with pytest.raises(dbapi.InternalError): db.cursor() # when the database is back, the cursor can be created as usual db._kwargs['database'] = 'ok' cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_cursor_in_transaction_when_connection_is_replaced(): """Check that a transaction is not silently continued elsewhere.""" db = steady_db_connect(dbapi, database='ok') db.begin() con = db._con db._con.valid = False with pytest.raises(dbapi.InternalError): db.cursor() # the connection has been replaced, but the error is still raised, # since the transaction cannot be continued on the new connection assert db._con is not con assert db._con.valid assert not db._transaction def test_cursor_in_transaction_when_reopening_fails(): """A transaction that cannot be reopened must not stay active. This is the counterpart of the test below for the case that already the creation of the cursor fails, and not only its execution. """ db = steady_db_connect(dbapi, database='ok') db.begin() assert db._transaction # the database goes away and cannot be reached again db._con.valid = False db._kwargs['database'] = 'error' with pytest.raises(dbapi.InternalError): db.cursor() assert not db._transaction # when the database is back, the failover mechanism must work again db._kwargs['database'] = 'ok' cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' assert db._con.valid def test_transaction_ended_when_reopening_fails(): """A transaction that cannot be reopened must not stay active. Otherwise the failover mechanism would stay suspended on the connection, so that it could never recover by itself again. """ db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() db.begin() assert db._transaction # the database goes away and cannot be reached again db._kwargs['database'] = 'error' db._con.valid = cursor._cursor.valid = False with pytest.raises(dbapi.InternalError): cursor.execute('select test') assert not db._transaction # when the database is back, the failover mechanism must work again db._kwargs['database'] = 'ok' cursor.execute('select test') assert cursor.fetchone() == 'test' assert db._con.valid def test_execute_when_reopening_fails(): """Check that a statement is reported when there is no recovery.""" db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() # the database goes away and cannot be reached again db._con.valid = cursor._cursor.valid = False db._kwargs['database'] = 'error' with pytest.raises(dbapi.InternalError): cursor.execute('select test') # when the database is back, the statement can be executed again db._kwargs['database'] = 'ok' cursor.execute('select test') assert cursor.fetchone() == 'test' def test_execute_when_cursor_cannot_be_reopened(): """Check that a statement is reported when no cursor can be opened.""" db = steady_db_connect(dbapi, database='ok') cursor = db.cursor() # the cursor cannot be recreated, neither on the old connection # that is reopened first, nor on a completely new connection cursor._args = ('error',) con = db._con db._con.valid = cursor._cursor.valid = False with pytest.raises(dbapi.InternalError): cursor.execute('select test') assert db._con is con assert not db._con.valid class FlakyCreator: """A creator that fails at the first attempt to reconnect.""" def __init__(self): """Create the creator with the failover switched off.""" self.fail = False def __call__(self, *args, **kwargs): """Create a mocked connection, but fail once when requested.""" if self.fail: self.fail = False raise dbapi.OperationalError return dbapi.connect(*args, **kwargs) def test_execute_retried_on_a_new_connection(): """Check that the failover falls back on a completely new connection.""" creator = FlakyCreator() db = steady_db_connect(creator, database='ok') cursor = db.cursor() cursor.execute('select test') assert db._usage == 1 con = db._con # the connection breaks, and reopening it while getting a new cursor # fails as well, so the whole connection is created from scratch db._con.valid = cursor._cursor.valid = False creator.fail = True cursor.execute('select test2') assert cursor.fetchone() == 'test2' assert db._con is not con assert db._con.valid assert not creator.fail # the usage count has been restarted together with the connection assert db._usage == 1 assert db._con.num_uses == 1 def test_execute_retried_on_a_new_connection_with_sizes(): """Check that the stored sizes are set on the new connection too.""" creator = FlakyCreator() db = steady_db_connect(creator, database='ok') cursor = db.cursor() cursor.setinputsizes([7, 42, 6]) cursor.setoutputsize(9) db._con.valid = cursor._cursor.valid = False creator.fail = True cursor.execute('get sizes') assert cursor.fetchone() == ([7, 42, 6], {None: 9}) # the sizes have been cleared after they have been used cursor.execute('get sizes') assert cursor.fetchone() == ([], {}) def timeout_is_not_fatal(error): """Treat a deliberate server side timeout as not fatal.""" return not error.args or error.args[0] != 3024 def always_fatal(error): # noqa: ARG001 """Treat every error as fatal for the connection.""" return True def never_fatal(error): # noqa: ARG001 """Treat no error as fatal for the connection.""" return False def test_connection_isfatal_must_be_callable(): """Check that isfatal must be a callable when it is given.""" steady_db_connect(dbapi, isfatal=None) steady_db_connect(dbapi, isfatal=always_fatal) with pytest.raises(TypeError): steady_db_connect(dbapi, isfatal='always') with pytest.raises(TypeError): steady_db_connect(dbapi, isfatal=42) def test_connection_isfatal_is_asked(): """Check that isfatal is asked and can veto the failover.""" errors = [] def record_and_veto(error): errors.append(error) return False db = steady_db_connect(dbapi, isfatal=record_and_veto) cursor = db.cursor() cursor.execute('select test') assert cursor.fetchone() == 'test' assert not errors # not asked as long as nothing fails con = db._con db.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test') assert len(errors) == 1 assert isinstance(errors[0], dbapi.InternalError) # the connection has not been replaced since the failover was vetoed assert db._con is con def test_connection_isfatal_false_and_true(): """Check that isfatal can switch the failover off and on.""" db = steady_db_connect(dbapi, isfatal=never_fatal) cursor = db.cursor() db.close() with pytest.raises(dbapi.InternalError): cursor.execute('select test') db = steady_db_connect(dbapi, isfatal=always_fatal) cursor = db.cursor() db.close() cursor.execute('select test') # transparently reopened as usual assert cursor.fetchone() == 'test' def test_connection_isfatal_selectively(): """Statement timeouts shall not be retried (see issue #58).""" dbapi.Connection.num_timeouts = 0 db = steady_db_connect(dbapi) cursor = db.cursor() with pytest.raises(dbapi.OperationalError): cursor.execute('timeout') # by default the statement is executed three times in total: # once directly, once with a new cursor, once with a new connection assert dbapi.Connection.num_timeouts == 3 dbapi.Connection.num_timeouts = 0 db = steady_db_connect(dbapi, isfatal=timeout_is_not_fatal) cursor = db.cursor() with pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 # the connection is still usable and the failed statement was not counted assert db._usage == 0 cursor.execute('select test') assert cursor.fetchone() == 'test' assert db._usage == 1 # other failures are still handled by the failover mechanism db.close() cursor.execute('select test') assert cursor.fetchone() == 'test' def test_connection_isfatal_cursor_creation(): """Check that isfatal also applies when creating a cursor.""" db = steady_db_connect(dbapi, isfatal=never_fatal) con = db._con db.close() with pytest.raises(dbapi.InternalError): db.cursor() assert db._con is con db = steady_db_connect(dbapi, isfatal=always_fatal) con = db._con db.close() db.cursor() # transparently reopened as usual assert db._con is not con def test_connection_isfatal_ignored_when_overused(): """The reset triggered by maxusage must not be vetoed.""" db = steady_db_connect(dbapi, 3, isfatal=never_fatal) cursor = db.cursor() for i in range(10): cursor.execute(f'select test{i}') assert cursor.fetchone() == f'test{i}' assert db._con.valid assert db._usage == i % 3 + 1 assert db._con.num_uses == i % 3 + 1 def test_cursor_no_failover(): """Check that no_failover() suspends the failover mechanism.""" dbapi.Connection.num_timeouts = 0 db = steady_db_connect(dbapi) cursor = db.cursor() with cursor.no_failover(), pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 assert cursor._no_failover == 0 # outside of the context, the failover mechanism is active again dbapi.Connection.num_timeouts = 0 with pytest.raises(dbapi.OperationalError): cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 3 def test_cursor_no_failover_keeps_bookkeeping(): """Check that no_failover() still counts the usage.""" db = steady_db_connect(dbapi) cursor = db.cursor() with cursor.no_failover(): cursor.execute('select test') assert cursor.fetchone() == 'test' # in contrast to using the raw cursor, the usage is still counted assert db._usage == 1 assert db._con.num_uses == 1 def test_cursor_no_failover_broken_connection(): """Check that no_failover() does not replace the connection.""" db = steady_db_connect(dbapi) cursor = db.cursor() con = db._con db.close() with cursor.no_failover(), pytest.raises(dbapi.InternalError): cursor.execute('select test') assert db._con is con # not replaced inside the context # afterwards the failover mechanism works again cursor.execute('select test') assert cursor.fetchone() == 'test' assert db._con is not con def test_cursor_no_failover_nested(): """Check that no_failover() contexts can be nested.""" db = steady_db_connect(dbapi) cursor = db.cursor() assert cursor._no_failover == 0 with cursor.no_failover() as cur: assert cur is cursor assert cursor._no_failover == 1 with cursor.no_failover(): assert cursor._no_failover == 2 assert cursor._no_failover == 1 assert cursor._no_failover == 0 # the counter is also restored when the statement fails with pytest.raises(dbapi.OperationalError), cursor.no_failover(): cursor.execute('timeout') assert cursor._no_failover == 0 def test_dbapi_connection(): """Check that the underlying DB-API 2 connection is accessible.""" db = steady_db_connect(dbapi) con = db.dbapi_connection assert isinstance(con, dbapi.Connection) assert con is db._con db.close() db.cursor() # transparently reopened assert db.dbapi_connection is db._con assert db.dbapi_connection is not con def test_dbapi_cursor(): """Check that the underlying DB-API 2 cursor is accessible.""" db = steady_db_connect(dbapi) cursor = db.cursor() raw_cursor = cursor.dbapi_cursor assert isinstance(raw_cursor, dbapi.Cursor) assert raw_cursor is cursor._cursor # the raw cursor bypasses the failover mechanism and the bookkeeping dbapi.Connection.num_timeouts = 0 with pytest.raises(dbapi.OperationalError): raw_cursor.execute('timeout') assert dbapi.Connection.num_timeouts == 1 raw_cursor.execute('select test') assert raw_cursor.fetchone() == 'test' assert db._con.num_uses == 2 # both statements reached the server assert db._usage == 0 # but they were not counted by the steady connection cursor._cursor = None with pytest.raises(InvalidCursorError): cursor.dbapi_cursor # noqa: B018 def test_invalid_cursor(): """Check that the members of an invalid cursor cannot be accessed.""" db = steady_db_connect(dbapi) cursor = db.cursor() assert cursor.fetchone() is None # inherited from the underlying cursor assert callable(cursor.execute) # made "tough" by the wrapper cursor._cursor = None with pytest.raises(InvalidCursorError): cursor.fetchone # noqa: B018 with pytest.raises(InvalidCursorError): cursor.execute # noqa: B018 WebwareForPython-DBUtils-8de754c/tests/test_steady_pg.py000066400000000000000000000363321524214146500234550ustar00rootroot00000000000000"""Test the SteadyPg module. Note: We do not test the real PyGreSQL module, but we just mock the basic connection functionality of that module. We assume that the PyGreSQL module will detect lost connections correctly and set the status flag accordingly. Copyright and credit info: * This test was contributed by Christoph Zwerschke """ import sys import pg import pytest from dbutils.steady_pg import InvalidConnectionError, SteadyPgConnection def test_version(): """Check that the module and class versions are in sync.""" from dbutils import __version__, steady_pg assert steady_pg.__version__ == __version__ assert steady_pg.SteadyPgConnection.version == __version__ def test_mocked_connection(): """Check that the mocked PyGreSQL connection behaves as expected.""" db_cls = pg.DB db = db_cls( 'SteadyPgTestDB', user='SteadyPgTestUser') assert hasattr(db, 'db') assert hasattr(db.db, 'status') assert db.db.status assert hasattr(db.db, 'query') assert hasattr(db.db, 'close') assert not hasattr(db.db, 'reopen') assert hasattr(db, 'reset') assert hasattr(db.db, 'num_queries') assert hasattr(db.db, 'session') assert not hasattr(db.db, 'get_tables') assert hasattr(db.db, 'db') assert db.db.db == 'SteadyPgTestDB' assert hasattr(db.db, 'user') assert db.db.user == 'SteadyPgTestUser' assert hasattr(db, 'query') assert hasattr(db, 'close') assert hasattr(db, 'reopen') assert hasattr(db, 'reset') assert hasattr(db, 'num_queries') assert hasattr(db, 'session') assert hasattr(db, 'get_tables') assert hasattr(db, 'dbname') assert db.dbname == 'SteadyPgTestDB' assert hasattr(db, 'user') assert db.user == 'SteadyPgTestUser' # queries are counted, and reopening the connection resets the counter for i in range(3): assert db.num_queries == i assert db.query(f'select test{i}') == f'test{i}' assert db.db.status db.reopen() assert db.db.status assert db.num_queries == 0 assert db.query('select test4') == 'test4' assert db.get_tables() == 'test' # a closed connection cannot be used or closed again db.close() try: status = db.db.status except AttributeError: status = False assert not status with pytest.raises(pg.InternalError): db.close() with pytest.raises(pg.InternalError): db.query('select test') with pytest.raises(pg.InternalError): db.get_tables() def test_broken_connection(): """Check that errors when opening a connection are reported.""" with pytest.raises(TypeError): SteadyPgConnection('wrong') db = SteadyPgConnection(dbname='ok') internal_error_cls = sys.modules[db._con.__module__].InternalError for _i in range(3): db.close() del db with pytest.raises(internal_error_cls): SteadyPgConnection(dbname='error') @pytest.mark.parametrize("closeable", [False, True]) def test_close(closeable): """Check that closing is only allowed when the connection is closeable.""" db = SteadyPgConnection(closeable=closeable) assert db._con.db assert db._con.valid is True db.close() assert closeable ^ (db._con.db is not None and db._con.valid) db.close() assert closeable ^ (db._con.db is not None and db._con.valid) db._close() assert not db._con.db db._close() assert not db._con.db def test_connection(): """Check that the connection wraps the PyGreSQL connection.""" db = SteadyPgConnection( 0, None, 1, 'SteadyPgTestDB', user='SteadyPgTestUser') assert hasattr(db, 'db') assert hasattr(db, '_con') assert db.db == db._con.db assert hasattr(db, '_usage') assert db._usage == 0 assert hasattr(db.db, 'status') assert db.db.status assert hasattr(db.db, 'query') assert hasattr(db.db, 'close') assert not hasattr(db.db, 'reopen') assert hasattr(db.db, 'reset') assert hasattr(db.db, 'num_queries') assert hasattr(db.db, 'session') assert hasattr(db.db, 'db') assert db.db.db == 'SteadyPgTestDB' assert hasattr(db.db, 'user') assert db.db.user == 'SteadyPgTestUser' assert not hasattr(db.db, 'get_tables') assert hasattr(db, 'query') assert hasattr(db, 'close') assert hasattr(db, 'reopen') assert hasattr(db, 'reset') assert hasattr(db, 'num_queries') assert hasattr(db, 'session') assert hasattr(db, 'dbname') assert db.dbname == 'SteadyPgTestDB' assert hasattr(db, 'user') assert db.user == 'SteadyPgTestUser' assert hasattr(db, 'get_tables') # the wrapper keeps its own usage count next to the query count, # and get_tables() counts as a use, but not as a query for i in range(3): assert db._usage == i assert db.num_queries == i assert db.query(f'select test{i}') == f'test{i}' assert db.db.status assert db.get_tables() == 'test' assert db.db.status assert db._usage == 4 assert db.num_queries == 3 # reopening the connection resets both counters db.reopen() assert db.db.status assert db._usage == 0 assert db.num_queries == 0 assert db.query('select test') == 'test' assert db.db.status assert hasattr(db._con, 'status') assert db._con.status def test_connection_close(): """Check that closing the connection closes the PyGreSQL connection.""" db = SteadyPgConnection( 0, None, 1, 'SteadyPgTestDB', user='SteadyPgTestUser') db.query('select test') assert db.db.status db.close() try: status = db.db.status except AttributeError: status = False assert not status # the underlying connection is still wrapped, but cannot be used any more assert hasattr(db._con, 'close') assert hasattr(db._con, 'query') internal_error_cls = sys.modules[db._con.__module__].InternalError with pytest.raises(internal_error_cls): db._con.close() with pytest.raises(internal_error_cls): db._con.query('select test') def test_connection_reopen_when_broken(): """Check that the PyGreSQL connection is transparently reopened.""" db = SteadyPgConnection( 0, None, 1, 'SteadyPgTestDB', user='SteadyPgTestUser') # querying a closed connection reopens it and starts counting anew db.close() assert db.query('select test') == 'test' assert db.db.status assert db._usage == 1 assert db.num_queries == 1 # the same happens when the connection reports a bad status db.db.status = False assert not db.db.status assert db.query('select test') == 'test' assert db.db.status assert db._usage == 1 assert db.num_queries == 1 # and when it is used through one of the PyGreSQL extension methods db.db.status = False assert not db.db.status assert db.get_tables() == 'test' assert db.db.status assert db._usage == 1 assert db.num_queries == 0 def test_connection_context_handler(): """Check that the context handler commits or rolls back.""" db = SteadyPgConnection( 0, None, 1, 'SteadyPgTestDB', user='SteadyPgTestUser') assert db.session == [] with db: db.query('select test') assert db.session == ['begin', 'commit'] try: with db: db.query('error') except pg.ProgrammingError: error = True else: error = False assert error assert db._con.session == ['begin', 'commit', 'begin', 'rollback'] def test_connection_maxusage(): """Check that the connection is reset when used too often.""" db = SteadyPgConnection(10) # the connection is silently reset after every 10 uses for i in range(100): r = db.query(f'select test{i}') assert r == f'test{i}' assert db.db.status j = i % 10 + 1 assert db._usage == j assert db.num_queries == j # inside a transaction the reset is postponed until it has been ended, # so the usage count keeps growing beyond the maximum until then db.begin() for i in range(100): r = db.get_tables() assert r == 'test' assert db.db.status if i == 49: db.commit() j = i % 10 + 1 if i > 49 else i + 11 assert db._usage == j j = 0 if i > 49 else 10 assert db.num_queries == j # a connection that breaks in between is reset early, # which also restarts the counting of the uses for i in range(10): if i == 7: db.db.status = False r = db.query(f'select test{i}') assert r == f'test{i}' j = i % 7 + 1 assert db._usage == j assert db.num_queries == j for i in range(10): if i == 5: db.db.status = False r = db.get_tables() assert r == 'test' j = (i + (3 if i < 5 else -5)) % 10 + 1 assert db._usage == j j = 3 if i < 5 else 0 assert db.num_queries == j # closing and reopening the connection restart the counting as well db.close() assert db.query('select test1') == 'test1' assert db._usage == 1 assert db.num_queries == 1 db.reopen() assert db._usage == 0 assert db.num_queries == 0 assert db.query('select test2') == 'test2' assert db._usage == 1 assert db.num_queries == 1 def test_connection_setsession(): """Check that the session is prepared after every reopening.""" db = SteadyPgConnection(3, ('set time zone', 'set datestyle')) assert hasattr(db, 'num_queries') assert db.num_queries == 0 assert hasattr(db, 'session') assert tuple(db.session) == ('time zone', 'datestyle') for _i in range(11): db.query('select test') assert db.num_queries == 2 assert db.session == ['time zone', 'datestyle'] db.query('set test') assert db.num_queries == 2 assert db.session == ['time zone', 'datestyle', 'test'] db.query('select test') assert db.num_queries == 1 assert db.session == ['time zone', 'datestyle'] db.close() db.query('set test') assert db.num_queries == 0 assert db.session == ['time zone', 'datestyle', 'test'] @pytest.mark.parametrize("closeable", [False, True]) def test_begin(closeable): """Check that transparent reopening is suspended after begin().""" db = SteadyPgConnection(closeable=closeable) db.begin() assert db.session == ['begin'] db.query('select test') assert db.num_queries == 1 db.close() db.query('select test') assert db.num_queries == 1 db.begin() assert db.session == ['begin'] db.db.close() with pytest.raises(pg.InternalError): db.query('select test') assert db.num_queries == 0 db.query('select test') assert db.num_queries == 1 assert db.begin('select sql:begin') == 'sql:begin' assert db.num_queries == 2 def test_query_error_when_connection_is_healthy(): """Check that a query error is propagated when the connection is fine.""" db = SteadyPgConnection() db.query('select test') assert db._usage == 1 # the statement is bad, but the connection is not, so it is not reset with pytest.raises(pg.ProgrammingError): db.query('error') assert db.db.status assert db._usage == 1 assert db.query('select test') == 'test' assert db._usage == 2 def test_query_retried_when_connection_is_lost(): """Check that a query is retried when the connection was lost.""" db = SteadyPgConnection() query = db._con.query queries = [] def failing_query(qstr): """Lose the connection while the first query is running.""" queries.append(qstr) if len(queries) == 1: db._con.db.status = False raise pg.InternalError return query(qstr) db._con.query = failing_query # the connection looked healthy before the query, so it is only # reset afterwards, and the query is then run a second time assert db.query('select test') == 'test' assert queries == ['select test', 'select test'] assert db.db.status assert db._usage == 1 def test_reopen_when_broken_in_transaction(): """Check that a transaction is ended when reopening fails.""" db = SteadyPgConnection() db.begin() assert db._transaction # the connection cannot even be closed properly any more db._con.db.valid = False db.reopen() assert not db._transaction def test_reset_when_reopening_fails(): """Check that a rollback is attempted when a reset is impossible.""" db = SteadyPgConnection() db.begin() # the connection can neither be reset nor have its session prepared db._setsession_sql = ('error',) db._con.db = None db.reset() assert not db._transaction assert db.session == ['rollback'] def test_invalid_connection(): """Check that the members of an invalid connection are not accessible.""" db = SteadyPgConnection() assert db.query('select test') == 'test' db._con = None with pytest.raises(InvalidConnectionError): db.query # noqa: B018 with pytest.raises(InvalidConnectionError): db.session # noqa: B018 @pytest.fixture def pg_with_transaction_methods(monkeypatch): """Add the transaction methods that real PyGreSQL connections have.""" def transaction_method(name): """Create the transaction method with the given name.""" def method(self, sql=None): """Run the given or the default transaction command.""" return self.query(sql or name) return method for name in ('begin', 'end', 'commit', 'rollback'): monkeypatch.setattr( pg.DB, name, transaction_method(name), raising=False) @pytest.mark.usefixtures("pg_with_transaction_methods") @pytest.mark.parametrize("end", ["end", "commit", "rollback"]) def test_transaction_methods_of_driver(end): """Check that the transaction methods of the driver are used.""" db = SteadyPgConnection() db.begin() assert db._transaction assert db.session == ['begin'] db.query('select test') assert db.num_queries == 1 getattr(db, end)() assert not db._transaction assert db.session == ['begin', end] @pytest.mark.usefixtures("pg_with_transaction_methods") @pytest.mark.parametrize("end", ["end", "commit", "rollback"]) def test_transaction_methods_of_driver_with_sql(end): """Check that explicit SQL commands are passed on to the driver.""" db = SteadyPgConnection() assert db.begin(f'select sql:{end}') == f'sql:{end}' assert db._transaction assert db.num_queries == 1 assert getattr(db, end)(f'select sql:{end}') == f'sql:{end}' assert not db._transaction assert db.num_queries == 2 @pytest.mark.parametrize("closeable", [False, True]) @pytest.mark.parametrize("end", ["end", "commit", "rollback"]) def test_end(closeable, end): """Check that end(), commit() and rollback() finish the transaction.""" db = SteadyPgConnection(closeable=closeable) db.begin() db.query('select test') getattr(db, end)() assert db.session == ['begin', end] # the transaction is over, so the connection is reopened transparently db.db.close() db.query('select test') assert db.num_queries == 1 # a transaction can also be ended with an explicit SQL command db.begin() assert db._transaction assert getattr(db, end)(f'select sql:{end}') == f'sql:{end}' assert not db._transaction assert db.num_queries == 2 WebwareForPython-DBUtils-8de754c/tests/test_threading_local.py000066400000000000000000000040721524214146500246110ustar00rootroot00000000000000"""Test the ThreadingLocal module.""" from threading import Thread from dbutils.persistent_db import local def test_getattr(): """Check that attributes can be set and read back.""" my_data = local() my_data.number = 42 assert my_data.number == 42 def test_dict(): """Check that the attributes are exposed via the instance dict.""" my_data = local() my_data.number = 42 assert my_data.__dict__ == {'number': 42} my_data.__dict__.setdefault('widgets', []) assert my_data.widgets == [] def test_threadlocal(): """Check that each thread sees only its own attributes.""" def f(): items = sorted(my_data.__dict__.items()) log.append(items) my_data.number = 11 log.append(my_data.number) my_data = local() my_data.number = 42 log = [] thread = Thread(target=f) thread.start() thread.join() assert log == [[], 11] assert my_data.number == 42 def test_subclass(): """Check that subclasses with own attributes and slots work.""" class MyLocal(local): number = 2 initialized = 0 def __init__(self, **kw): if self.initialized: raise SystemError self.initialized = 1 self.__dict__.update(kw) def squared(self): return self.number ** 2 my_data = MyLocal(color='red') assert my_data.number == 2 assert my_data.color == 'red' del my_data.color assert my_data.squared() == 4 def f(): items = sorted(my_data.__dict__.items()) log.append(items) my_data.number = 7 log.append(my_data.number) log = [] thread = Thread(target=f) thread.start() thread.join() assert log == [[('color', 'red'), ('initialized', 1)], 7] assert my_data.number == 2 assert not hasattr(my_data, 'color') class MyLocal(local): __slots__ = ('number',) my_data = MyLocal() my_data.number = 42 my_data.color = 'red' thread = Thread(target=f) thread.start() thread.join() assert my_data.number == 7 WebwareForPython-DBUtils-8de754c/tox.ini000066400000000000000000000013401524214146500202250ustar00rootroot00000000000000[tox] envlist = py3{9,10,11,12,13,14}, ruff, manifest, docs, spell [testenv:py37] skip_install = true deps = dist/dbutils-3.2.0-py3-none-any.whl[tests] commands = pytest {posargs} [testenv:py38] skip_install = true deps = dist/dbutils-3.2.0-py3-none-any.whl[tests] commands = pytest {posargs} [testenv] setenv = PYTHONPATH = {toxinidir} extras = tests commands = pytest {posargs} [testenv:spell] basepython = python3.14 deps = codespell commands = codespell . [testenv:ruff] basepython = python3.14 deps = ruff commands = ruff check . [testenv:manifest] basepython = python3.14 deps = check-manifest commands = check-manifest -v [testenv:docs] basepython = python3.14 extras = docs changedir = docs commands = python make.py