././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5172565 requests_futures-1.1.0/0000755000175000017500000000000015244425101013735 5ustar00rossross././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5886335 requests_futures-1.1.0/LICENSE0000644000175000017500000000110615244416174014752 0ustar00rossrossCopyright 2013 Ross McFarland Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5886335 requests_futures-1.1.0/MANIFEST.in0000644000175000017500000000013115244416174015500 0ustar00rossrossinclude README.rst include LICENSE include requirements.txt recursive-include tests *.py ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5162578 requests_futures-1.1.0/PKG-INFO0000644000175000017500000002240615244425101015036 0ustar00rossrossMetadata-Version: 2.4 Name: requests-futures Version: 1.1.0 Summary: Asynchronous Python HTTP for Humans. Author-email: Ross McFarland License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/ross/requests-futures Project-URL: Source, https://github.com/ross/requests-futures Project-URL: Issues, https://github.com/ross/requests-futures/issues Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Natural Language :: English Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Requires-Python: >=3.10 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: requests>=1.2.0 Provides-Extra: test Requires-Dist: pytest>=6.2.5; extra == "test" Requires-Dist: pytest-cov>=3.0.0; extra == "test" Requires-Dist: pytest-httpbin>=2.0.0; extra == "test" Provides-Extra: dev Requires-Dist: pytest>=6.2.5; extra == "dev" Requires-Dist: pytest-cov>=3.0.0; extra == "dev" Requires-Dist: pytest-httpbin>=2.0.0; extra == "dev" Requires-Dist: black<27.0.0,>=26.0.0; extra == "dev" Requires-Dist: build>=0.7.0; extra == "dev" Requires-Dist: changelet; extra == "dev" Requires-Dist: isort>=5.11.4; extra == "dev" Requires-Dist: proviso; extra == "dev" Requires-Dist: pyflakes>=2.2.0; extra == "dev" Requires-Dist: readme_renderer[rst]>=26.0; extra == "dev" Requires-Dist: twine>=3.4.2; extra == "dev" Provides-Extra: docs Requires-Dist: sphinx>=8.1.0; extra == "docs" Requires-Dist: sphinx-copybutton>=0.5.2; extra == "docs" Requires-Dist: sphinx_rtd_theme; extra == "docs" Dynamic: license-file Asynchronous Python HTTP Requests for Humans ============================================ .. image:: https://readthedocs.org/projects/requests-futures/badge/?version=latest :target: https://requests-futures.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status Small add-on for the python requests_ http library that makes use of the standard library's `concurrent.futures`_. Requires Python 3.10 or newer. The additional API and changes are minimal and strives to avoid surprises. The following synchronous code: .. code-block:: python from requests import Session session = Session() # first requests starts and blocks until finished response_one = session.get('http://httpbin.org/get') # second request starts once first is finished response_two = session.get('http://httpbin.org/get?foo=bar') # both requests are complete print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) Can be translated to make use of futures, and thus be asynchronous by creating a FuturesSession and catching the returned Future in place of Response. The Response can be retrieved by calling the result method on the Future: .. code-block:: python from requests_futures.sessions import FuturesSession session = FuturesSession() # first request is started in background future_one = session.get('http://httpbin.org/get') # second requests is started immediately future_two = session.get('http://httpbin.org/get?foo=bar') # wait for the first request to complete, if it hasn't already response_one = future_one.result() print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) # wait for the second request to complete, if it hasn't already response_two = future_two.result() print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) By default a ThreadPoolExecutor is created with 8 workers. That default executor is a single ``requests.Session`` shared across all of its worker threads, and ``Session`` is not thread-safe: concurrently mutating shared state on it, such as the cookie jar or ``session.headers``, from multiple in-flight requests is a real hazard. See the `full thread-safety discussion`_ in the docs for what to do instead. That's it. The api of requests.Session is preserved without any modifications beyond returning a Future rather than Response. As with all futures exceptions are shifted (thrown) to the future.result() call so try/except blocks should be moved there. Tying extra information to the request/response =============================================== The most common piece of information needed is the URL of the request. This can be accessed without any extra steps using the `request` property of the response object. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[session.get(f'http://httpbin.org/get?{i}') for i in range(3)] for future in as_completed(futures): resp = future.result() pprint({ 'url': resp.request.url, 'content': resp.json(), }) There are situations in which you may want to tie additional information to a request/response. There are a number of ways to go about this, the simplest is to attach additional information to the future object itself. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[] for i in range(3): future = session.get('http://httpbin.org/get') future.i = i futures.append(future) for future in as_completed(futures): resp = future.result() pprint({ 'i': future.i, 'content': resp.json(), }) Canceling queued requests (a.k.a cleaning up after yourself) ============================================================ If you know that you won't be needing any additional responses from futures that haven't yet resolved, it's a good idea to cancel those requests. You can do this by using the session as a context manager: .. code-block:: python from requests_futures.sessions import FuturesSession with FuturesSession(max_workers=1) as session: future = session.get('https://httpbin.org/get') future2 = session.get('https://httpbin.org/delay/10') future3 = session.get('https://httpbin.org/delay/10') response = future.result() Exiting the ``with`` block calls ``close()``, which shuts down the session's own executor with ``cancel_futures=True``: every queued request that hasn't started running yet is cancelled immediately, not merely skipped one at a time. Here, with ``max_workers=1``, ``future`` is already done by the time the block exits, but neither ``future2`` nor ``future3`` has started, so both are cancelled together, saving the time and resources their requests would otherwise have used. A request that is already running when ``close()`` runs is left to finish. Using ProcessPoolExecutor ========================= A ``ProcessPoolExecutor`` can be supplied in place of a ``ThreadPoolExecutor`` to run requests in separate processes instead of threads, which is useful when per-request memory usage is high enough that cycling the interpreter is needed to release it back to the OS. Everything submitted to a process pool must be picklable, which means any ``FuturesSession`` subclass used this way must be importable at module scope (not defined inline, e.g. in a function or ``__main__`` script body). If something in the request isn't picklable, ``FuturesSession`` raises a ``RuntimeError`` pointing back to this documentation rather than letting the pickling failure surface on its own. .. code-block:: python from concurrent.futures import ProcessPoolExecutor from requests_futures.sessions import FuturesSession session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10)) # ... use as before See the `full ProcessPoolExecutor guide`_ in the docs for a module-global callback example and more on the pickling requirements. Documentation ============= Full documentation, including everything below, lives at https://requests-futures.readthedocs.io/: * ``as_completed`` with a timeout * error handling across the future boundary (submit-time vs. ``result()``-time) * retries via a mounted ``HTTPAdapter`` and urllib3's ``Retry`` * sizing ``max_workers`` against the connection pool, including with a supplied ``session=`` * streaming responses (``stream=True``) and why it interacts badly with background work * sharing one executor across several sessions, and what ``close()`` does in that configuration * ``hooks``, the recommended replacement for the deprecated ``background_callback`` * the full ``ProcessPoolExecutor`` guide * the full thread-safety discussion Installation ============ pip install requests-futures .. _`requests`: https://github.com/kennethreitz/requests .. _`concurrent.futures`: http://docs.python.org/dev/library/concurrent.futures.html .. _`full thread-safety discussion`: https://requests-futures.readthedocs.io/en/latest/usage.html .. _`full ProcessPoolExecutor guide`: https://requests-futures.readthedocs.io/en/latest/usage.html ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5886335 requests_futures-1.1.0/README.rst0000644000175000017500000001672315244416174015447 0ustar00rossrossAsynchronous Python HTTP Requests for Humans ============================================ .. image:: https://readthedocs.org/projects/requests-futures/badge/?version=latest :target: https://requests-futures.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status Small add-on for the python requests_ http library that makes use of the standard library's `concurrent.futures`_. Requires Python 3.10 or newer. The additional API and changes are minimal and strives to avoid surprises. The following synchronous code: .. code-block:: python from requests import Session session = Session() # first requests starts and blocks until finished response_one = session.get('http://httpbin.org/get') # second request starts once first is finished response_two = session.get('http://httpbin.org/get?foo=bar') # both requests are complete print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) Can be translated to make use of futures, and thus be asynchronous by creating a FuturesSession and catching the returned Future in place of Response. The Response can be retrieved by calling the result method on the Future: .. code-block:: python from requests_futures.sessions import FuturesSession session = FuturesSession() # first request is started in background future_one = session.get('http://httpbin.org/get') # second requests is started immediately future_two = session.get('http://httpbin.org/get?foo=bar') # wait for the first request to complete, if it hasn't already response_one = future_one.result() print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) # wait for the second request to complete, if it hasn't already response_two = future_two.result() print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) By default a ThreadPoolExecutor is created with 8 workers. That default executor is a single ``requests.Session`` shared across all of its worker threads, and ``Session`` is not thread-safe: concurrently mutating shared state on it, such as the cookie jar or ``session.headers``, from multiple in-flight requests is a real hazard. See the `full thread-safety discussion`_ in the docs for what to do instead. That's it. The api of requests.Session is preserved without any modifications beyond returning a Future rather than Response. As with all futures exceptions are shifted (thrown) to the future.result() call so try/except blocks should be moved there. Tying extra information to the request/response =============================================== The most common piece of information needed is the URL of the request. This can be accessed without any extra steps using the `request` property of the response object. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[session.get(f'http://httpbin.org/get?{i}') for i in range(3)] for future in as_completed(futures): resp = future.result() pprint({ 'url': resp.request.url, 'content': resp.json(), }) There are situations in which you may want to tie additional information to a request/response. There are a number of ways to go about this, the simplest is to attach additional information to the future object itself. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[] for i in range(3): future = session.get('http://httpbin.org/get') future.i = i futures.append(future) for future in as_completed(futures): resp = future.result() pprint({ 'i': future.i, 'content': resp.json(), }) Canceling queued requests (a.k.a cleaning up after yourself) ============================================================ If you know that you won't be needing any additional responses from futures that haven't yet resolved, it's a good idea to cancel those requests. You can do this by using the session as a context manager: .. code-block:: python from requests_futures.sessions import FuturesSession with FuturesSession(max_workers=1) as session: future = session.get('https://httpbin.org/get') future2 = session.get('https://httpbin.org/delay/10') future3 = session.get('https://httpbin.org/delay/10') response = future.result() Exiting the ``with`` block calls ``close()``, which shuts down the session's own executor with ``cancel_futures=True``: every queued request that hasn't started running yet is cancelled immediately, not merely skipped one at a time. Here, with ``max_workers=1``, ``future`` is already done by the time the block exits, but neither ``future2`` nor ``future3`` has started, so both are cancelled together, saving the time and resources their requests would otherwise have used. A request that is already running when ``close()`` runs is left to finish. Using ProcessPoolExecutor ========================= A ``ProcessPoolExecutor`` can be supplied in place of a ``ThreadPoolExecutor`` to run requests in separate processes instead of threads, which is useful when per-request memory usage is high enough that cycling the interpreter is needed to release it back to the OS. Everything submitted to a process pool must be picklable, which means any ``FuturesSession`` subclass used this way must be importable at module scope (not defined inline, e.g. in a function or ``__main__`` script body). If something in the request isn't picklable, ``FuturesSession`` raises a ``RuntimeError`` pointing back to this documentation rather than letting the pickling failure surface on its own. .. code-block:: python from concurrent.futures import ProcessPoolExecutor from requests_futures.sessions import FuturesSession session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10)) # ... use as before See the `full ProcessPoolExecutor guide`_ in the docs for a module-global callback example and more on the pickling requirements. Documentation ============= Full documentation, including everything below, lives at https://requests-futures.readthedocs.io/: * ``as_completed`` with a timeout * error handling across the future boundary (submit-time vs. ``result()``-time) * retries via a mounted ``HTTPAdapter`` and urllib3's ``Retry`` * sizing ``max_workers`` against the connection pool, including with a supplied ``session=`` * streaming responses (``stream=True``) and why it interacts badly with background work * sharing one executor across several sessions, and what ``close()`` does in that configuration * ``hooks``, the recommended replacement for the deprecated ``background_callback`` * the full ``ProcessPoolExecutor`` guide * the full thread-safety discussion Installation ============ pip install requests-futures .. _`requests`: https://github.com/kennethreitz/requests .. _`concurrent.futures`: http://docs.python.org/dev/library/concurrent.futures.html .. _`full thread-safety discussion`: https://requests-futures.readthedocs.io/en/latest/usage.html .. _`full ProcessPoolExecutor guide`: https://requests-futures.readthedocs.io/en/latest/usage.html ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5895298 requests_futures-1.1.0/pyproject.toml0000644000175000017500000000362615244416174016672 0ustar00rossross[build-system] requires = ["setuptools>=77"] build-backend = "setuptools.build_meta" [project] name = "requests-futures" description = "Asynchronous Python HTTP for Humans." authors = [{name = "Ross McFarland", email = "rwmcfa1@gmail.com"}] readme = "README.rst" license = "Apache-2.0" license-files = ["LICENSE"] requires-python = ">=3.10" dynamic = ["version"] classifiers = [ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Natural Language :: English", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", ] dependencies = [ "requests>=1.2.0", ] [project.urls] Homepage = "https://github.com/ross/requests-futures" Source = "https://github.com/ross/requests-futures" Issues = "https://github.com/ross/requests-futures/issues" [project.optional-dependencies] test = [ "pytest>=6.2.5", "pytest-cov>=3.0.0", "pytest-httpbin>=2.0.0", ] dev = [ "pytest>=6.2.5", "pytest-cov>=3.0.0", "pytest-httpbin>=2.0.0", "black>=26.0.0,<27.0.0", "build>=0.7.0", "changelet", "isort>=5.11.4", "proviso", "pyflakes>=2.2.0", "readme_renderer[rst]>=26.0", "twine>=3.4.2", ] docs = [ "sphinx>=8.1.0", "sphinx-copybutton>=0.5.2", "sphinx_rtd_theme", ] [tool.setuptools.dynamic] version = {attr = "requests_futures.__version__"} [tool.setuptools.packages.find] include = ["requests_futures*"] [tool.black] line-length=80 skip-string-normalization=true skip-magic-trailing-comma=true target-version=["py310", "py311", "py312", "py313", "py314"] [tool.isort] profile = "black" known_first_party="requests_futures" line_length=80 [tool.pytest.ini_options] pythonpath = "." [tool.changelet] module = "requests_futures" ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5157056 requests_futures-1.1.0/requests_futures/0000755000175000017500000000000015244425101017365 5ustar00rossross././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963905.6216664 requests_futures-1.1.0/requests_futures/__init__.py0000644000175000017500000000067115244425002021502 0ustar00rossross# -*- coding: utf-8 -*- # Requests Futures """ async requests HTTP library ~~~~~~~~~~~~~~~~~~~~~~~~~~~ """ import logging from logging import NullHandler __title__ = 'requests-futures' __version__ = '1.1.0' __author__ = 'Ross McFarland' __license__ = 'Apache 2.0' __copyright__ = 'Copyright 2013 Ross McFarland' # Set default logging handler to avoid "No handler found" warnings. logging.getLogger(__name__).addHandler(NullHandler()) ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5899947 requests_futures-1.1.0/requests_futures/sessions.py0000644000175000017500000004171115244416174021623 0ustar00rossross# -*- coding: utf-8 -*- """ requests_futures ~~~~~~~~~~~~~~~~ This module provides a small add-on for the requests http library that runs requests in the background using Python's built-in ``concurrent.futures``. from requests_futures.sessions import FuturesSession session = FuturesSession() # request is run in the background future = session.get('http://httpbin.org/get') # ... do other stuff ... # wait for the request to complete, if it hasn't already response = future.result() print('response status: {0}'.format(response.status_code)) print(response.content) """ from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor, wait from functools import partial from pickle import dumps from threading import Lock from warnings import warn from requests import Session from requests.adapters import DEFAULT_POOLSIZE, DEFAULT_RETRIES, Retry def wrap(self, sup, background_callback, *args_, **kwargs_): """Runs `sup`, then feeds its response through `background_callback`. This has to be a module-level function, rather than a bound method or a closure, because it is what actually gets submitted to the executor: a bound method isn't picklable, so this would fail as soon as `executor` is a :class:`~concurrent.futures.ProcessPoolExecutor`. :param self: The `FuturesSession` (or subclass) instance the request was made through; passed on to `background_callback`. :param sup: The callable that performs the actual HTTP request, e.g. ``partial(Session.request, self)``. :param background_callback: Called as ``background_callback(session, response)``. Deprecated in favor of `hooks`. :param args_: Positional arguments forwarded to `sup`. :param kwargs_: Keyword arguments forwarded to `sup`. :returns: `background_callback`'s return value, unless it returns `None`, in which case the :class:`~requests.Response` from `sup` is returned instead -- so a callback that mutates the response in place (e.g. calling ``resp.json()`` to pre-parse it) doesn't need to return anything. """ resp = sup(*args_, **kwargs_) result = background_callback(self, resp) return resp if result is None else result def _configure_adapters(session, adapter_kwargs): """Apply `adapter_kwargs` to the adapters already mounted on `session`. Adapters are reconfigured in place, rather than replaced, so that a supplied session keeps its retry policy and any custom adapter behavior while still picking up the pool sizing. """ for prefix in ('https://', 'http://'): adapter = session.get_adapter(prefix) if not hasattr(adapter, 'init_poolmanager'): # not an HTTPAdapter, pool sizing doesn't apply, leave it alone continue if 'max_retries' in adapter_kwargs: max_retries = adapter_kwargs['max_retries'] # matches HTTPAdapter.__init__'s own handling if max_retries == DEFAULT_RETRIES: adapter.max_retries = Retry(0, read=False) else: adapter.max_retries = Retry.from_int(max_retries) pool_connections = adapter_kwargs.get( 'pool_connections', adapter._pool_connections ) pool_maxsize = adapter_kwargs.get('pool_maxsize', adapter._pool_maxsize) pool_block = adapter_kwargs.get('pool_block', adapter._pool_block) if (pool_connections, pool_maxsize, pool_block) == ( adapter._pool_connections, adapter._pool_maxsize, adapter._pool_block, ): # nothing to resize, leave the existing pools and their warm # connections alone continue adapter.init_poolmanager( pool_connections, pool_maxsize, block=pool_block ) # cached proxy managers were built with the old pool settings and # init_poolmanager() doesn't touch them, so drop them to be rebuilt # from the new settings on next use for proxy_manager in adapter.proxy_manager.values(): proxy_manager.clear() adapter.proxy_manager.clear() PICKLE_ERROR = ( 'Cannot pickle request. Refer to documentation: https://' 'github.com/ross/requests-futures/#using-processpoolexecutor' ) class FuturesSession(Session): """A :class:`requests.Session` that runs requests in the background. :meth:`request` (and therefore the ``get``/``post``/etc. verb methods) do not block: they submit the actual HTTP call to a :class:`~concurrent.futures.ThreadPoolExecutor` (the default) or a caller-supplied :class:`~concurrent.futures.Executor`, such as a :class:`~concurrent.futures.ProcessPoolExecutor`, and immediately return a :class:`~concurrent.futures.Future` in place of the usual :class:`requests.Response`. Call :meth:`~concurrent.futures.Future.result` on it to block for, and retrieve, the response -- or use it with :func:`concurrent.futures.as_completed` / :func:`concurrent.futures.wait` to work with several in flight requests at once. See :doc:`usage` for worked examples, including retries, streaming, sharing an executor across sessions, and the ``ProcessPoolExecutor`` case. """ def __init__( self, executor=None, max_workers=8, session=None, adapter_kwargs=None, *args, **kwargs, ): """Creates a FuturesSession. :param executor: The executor to submit requests to. Defaults to a new :class:`~concurrent.futures.ThreadPoolExecutor` sized by `max_workers`, owned by this session -- see :meth:`close`. Pass a :class:`~concurrent.futures.ProcessPoolExecutor` to run requests in worker processes instead of threads; see :doc:`usage` for what that requires (module-global, picklable callables and arguments). An executor supplied here, or shared across several `FuturesSession` instances, is never shut down by :meth:`close`. :type executor: concurrent.futures.Executor, optional :param max_workers: Number of worker threads to create for the default `ThreadPoolExecutor`. Ignored, along with the connection pool resizing described below, whenever `executor` is passed explicitly -- size the pool yourself in that case. :type max_workers: int, optional :param session: An existing :class:`requests.Session` to actually issue requests through, instead of this one. Useful for reusing an already-configured session (auth, headers, mounted adapters, a custom :class:`~requests.adapters.HTTPAdapter` subclass) across one or more `FuturesSession` instances that share an `executor`. When supplied, :meth:`close` never closes it, and futures are submitted directly to `executor` rather than tracked for cancellation, since ownership of the session's lifecycle stays with the caller. :type session: requests.Session, optional :param adapter_kwargs: Keyword arguments forwarded to :meth:`~requests.adapters.HTTPAdapter.init_poolmanager` -- typically `pool_connections`, `pool_maxsize`, `pool_block`, and `max_retries` -- applied to the adapters already mounted on whichever session will actually serve requests (`session`, if supplied, otherwise this one). This reconfigures the existing adapters in place rather than mounting new ones, so a supplied session's retry policy and any custom adapter subclass are preserved. When `executor` is not supplied and `max_workers` exceeds :data:`requests.adapters.DEFAULT_POOLSIZE`, the pool is sized to `max_workers` by default so the worker threads aren't throttled by urllib3's default pool size; `adapter_kwargs` is merged over that default and applies regardless of whether the executor is owned or supplied. :type adapter_kwargs: dict, optional """ _adapter_kwargs = {} super(FuturesSession, self).__init__(*args, **kwargs) self._owned_executor = executor is None self._pending_futures = set() self._pending_futures_lock = Lock() self._close_lock = Lock() self._closed = False if executor is None: executor = ThreadPoolExecutor(max_workers=max_workers) # set connection pool size equal to max_workers if needed if max_workers > DEFAULT_POOLSIZE: _adapter_kwargs.update( { 'pool_connections': max_workers, 'pool_maxsize': max_workers, } ) _adapter_kwargs.update(adapter_kwargs or {}) if _adapter_kwargs: # configure whichever session will actually serve requests: the # supplied one, if any, otherwise self. `self.session` isn't # assigned yet, so the `session` argument is used directly here. _configure_adapters(session or self, _adapter_kwargs) self.executor = executor self.session = session def request(self, *args, **kwargs): """Maintains the existing api for Session.request. Used by all of the higher level methods, e.g. Session.get. The background_callback param allows you to do some processing on the response in the background, e.g. call resp.json() so that json parsing happens in the background thread. It is deprecated; use `hooks` instead. When the executor is a `ProcessPoolExecutor`, the function *and* its arguments must all be picklable, since that's what gets sent to the worker process; this is verified up front so a bad `hooks` callable or file-like `data=` raises `RuntimeError` here rather than a raw pickling error out of the returned `Future`. This method itself never blocks and never raises for problems with the request -- connection errors, timeouts, and bad status codes all surface later, from the returned `Future`'s :meth:`~concurrent.futures.Future.result`. `RuntimeError` is raised directly from here instead, in three cases: * the pickling guard above; * after :meth:`close` on a session with the default, self-owned executor (no `executor=` supplied) -- `close()` shuts that executor down, so this method's call to :meth:`~concurrent.futures.Executor.submit` raises directly, with `concurrent.futures`' own message ("cannot schedule new futures after shutdown"). This applies whether or not `session=` was also supplied, since `close()` shuts down any executor it owns either way; * after :meth:`close` on a session built with a supplied `executor=` and no `session=` -- the only combination where `close()` leaves the executor itself running but tracks pending futures and rejects new ones itself, with its own message ("cannot schedule new futures after close"). A session built with both a supplied `executor=` and a supplied `session=` is never blocked by `close()` at all: `close()` only closes the `FuturesSession`'s own connections in that case, and leaves both the executor and the supplied session running. :rtype: concurrent.futures.Future """ if self.session: func = self.session.request else: # avoid calling super to not break pickled method func = partial(Session.request, self) background_callback = kwargs.pop('background_callback', None) if background_callback: # stacklevel 4: warn -> request -> requests' Session. -> # our -> caller warn( '`background_callback` is DEPRECATED. Use `hooks` instead. ' 'Will be removed in 2.0.', DeprecationWarning, stacklevel=4, ) func = partial(wrap, self, func, background_callback) if isinstance(self.executor, ProcessPoolExecutor): # verify the whole call can be pickled, not just the function; # the executor pickles these same objects when submitting, so # anything rejected here would have failed there too, but with # a raw error out of future.result() instead of a pointer to # the docs. Pickle's failure mode varies by object and python # version, so the catch is broad; the original is preserved as # __cause__. try: dumps((func, args, kwargs)) except Exception as e: raise RuntimeError(PICKLE_ERROR) from e if self._owned_executor or self.session: return self.executor.submit(func, *args, **kwargs) with self._pending_futures_lock: if self._closed: raise RuntimeError('cannot schedule new futures after close') future = self.executor.submit(func, *args, **kwargs) self._pending_futures.add(future) future.add_done_callback(self._remove_pending_future) return future def _remove_pending_future(self, future): with self._pending_futures_lock: self._pending_futures.discard(future) def close(self): with self._close_lock: if self._owned_executor: self.executor.shutdown(cancel_futures=True) elif not self.session: with self._pending_futures_lock: self._closed = True pending_futures = tuple(self._pending_futures) for future in pending_futures: future.cancel() wait(pending_futures) super(FuturesSession, self).close() return super(FuturesSession, self).close() def get(self, url, **kwargs): r""" Sends a GET request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).get(url, **kwargs) def options(self, url, **kwargs): r"""Sends a OPTIONS request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).options(url, **kwargs) def head(self, url, **kwargs): r"""Sends a HEAD request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).head(url, **kwargs) def post(self, url, data=None, json=None, **kwargs): r"""Sends a POST request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param json: (optional) json to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).post( url, data=data, json=json, **kwargs ) def put(self, url, data=None, **kwargs): r"""Sends a PUT request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).put(url, data=data, **kwargs) def patch(self, url, data=None, **kwargs): r"""Sends a PATCH request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param data: (optional) Dictionary, list of tuples, bytes, or file-like object to send in the body of the :class:`Request`. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).patch(url, data=data, **kwargs) def delete(self, url, **kwargs): r"""Sends a DELETE request. Returns :class:`Future` object. :param url: URL for the new :class:`Request` object. :param \*\*kwargs: Optional arguments that ``request`` takes. :rtype: concurrent.futures.Future """ return super(FuturesSession, self).delete(url, **kwargs) ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5160134 requests_futures-1.1.0/requests_futures.egg-info/0000755000175000017500000000000015244425101021057 5ustar00rossross././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5052578 requests_futures-1.1.0/requests_futures.egg-info/PKG-INFO0000644000175000017500000002240615244425101022160 0ustar00rossrossMetadata-Version: 2.4 Name: requests-futures Version: 1.1.0 Summary: Asynchronous Python HTTP for Humans. Author-email: Ross McFarland License-Expression: Apache-2.0 Project-URL: Homepage, https://github.com/ross/requests-futures Project-URL: Source, https://github.com/ross/requests-futures Project-URL: Issues, https://github.com/ross/requests-futures/issues Classifier: Development Status :: 5 - Production/Stable Classifier: Intended Audience :: Developers Classifier: Natural Language :: English Classifier: Programming Language :: Python :: 3 Classifier: Programming Language :: Python :: 3.10 Classifier: Programming Language :: Python :: 3.11 Classifier: Programming Language :: Python :: 3.12 Classifier: Programming Language :: Python :: 3.13 Classifier: Programming Language :: Python :: 3.14 Requires-Python: >=3.10 Description-Content-Type: text/x-rst License-File: LICENSE Requires-Dist: requests>=1.2.0 Provides-Extra: test Requires-Dist: pytest>=6.2.5; extra == "test" Requires-Dist: pytest-cov>=3.0.0; extra == "test" Requires-Dist: pytest-httpbin>=2.0.0; extra == "test" Provides-Extra: dev Requires-Dist: pytest>=6.2.5; extra == "dev" Requires-Dist: pytest-cov>=3.0.0; extra == "dev" Requires-Dist: pytest-httpbin>=2.0.0; extra == "dev" Requires-Dist: black<27.0.0,>=26.0.0; extra == "dev" Requires-Dist: build>=0.7.0; extra == "dev" Requires-Dist: changelet; extra == "dev" Requires-Dist: isort>=5.11.4; extra == "dev" Requires-Dist: proviso; extra == "dev" Requires-Dist: pyflakes>=2.2.0; extra == "dev" Requires-Dist: readme_renderer[rst]>=26.0; extra == "dev" Requires-Dist: twine>=3.4.2; extra == "dev" Provides-Extra: docs Requires-Dist: sphinx>=8.1.0; extra == "docs" Requires-Dist: sphinx-copybutton>=0.5.2; extra == "docs" Requires-Dist: sphinx_rtd_theme; extra == "docs" Dynamic: license-file Asynchronous Python HTTP Requests for Humans ============================================ .. image:: https://readthedocs.org/projects/requests-futures/badge/?version=latest :target: https://requests-futures.readthedocs.io/en/latest/?badge=latest :alt: Documentation Status Small add-on for the python requests_ http library that makes use of the standard library's `concurrent.futures`_. Requires Python 3.10 or newer. The additional API and changes are minimal and strives to avoid surprises. The following synchronous code: .. code-block:: python from requests import Session session = Session() # first requests starts and blocks until finished response_one = session.get('http://httpbin.org/get') # second request starts once first is finished response_two = session.get('http://httpbin.org/get?foo=bar') # both requests are complete print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) Can be translated to make use of futures, and thus be asynchronous by creating a FuturesSession and catching the returned Future in place of Response. The Response can be retrieved by calling the result method on the Future: .. code-block:: python from requests_futures.sessions import FuturesSession session = FuturesSession() # first request is started in background future_one = session.get('http://httpbin.org/get') # second requests is started immediately future_two = session.get('http://httpbin.org/get?foo=bar') # wait for the first request to complete, if it hasn't already response_one = future_one.result() print('response one status: {0}'.format(response_one.status_code)) print(response_one.content) # wait for the second request to complete, if it hasn't already response_two = future_two.result() print('response two status: {0}'.format(response_two.status_code)) print(response_two.content) By default a ThreadPoolExecutor is created with 8 workers. That default executor is a single ``requests.Session`` shared across all of its worker threads, and ``Session`` is not thread-safe: concurrently mutating shared state on it, such as the cookie jar or ``session.headers``, from multiple in-flight requests is a real hazard. See the `full thread-safety discussion`_ in the docs for what to do instead. That's it. The api of requests.Session is preserved without any modifications beyond returning a Future rather than Response. As with all futures exceptions are shifted (thrown) to the future.result() call so try/except blocks should be moved there. Tying extra information to the request/response =============================================== The most common piece of information needed is the URL of the request. This can be accessed without any extra steps using the `request` property of the response object. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[session.get(f'http://httpbin.org/get?{i}') for i in range(3)] for future in as_completed(futures): resp = future.result() pprint({ 'url': resp.request.url, 'content': resp.json(), }) There are situations in which you may want to tie additional information to a request/response. There are a number of ways to go about this, the simplest is to attach additional information to the future object itself. .. code-block:: python from concurrent.futures import as_completed from pprint import pprint from requests_futures.sessions import FuturesSession session = FuturesSession() futures=[] for i in range(3): future = session.get('http://httpbin.org/get') future.i = i futures.append(future) for future in as_completed(futures): resp = future.result() pprint({ 'i': future.i, 'content': resp.json(), }) Canceling queued requests (a.k.a cleaning up after yourself) ============================================================ If you know that you won't be needing any additional responses from futures that haven't yet resolved, it's a good idea to cancel those requests. You can do this by using the session as a context manager: .. code-block:: python from requests_futures.sessions import FuturesSession with FuturesSession(max_workers=1) as session: future = session.get('https://httpbin.org/get') future2 = session.get('https://httpbin.org/delay/10') future3 = session.get('https://httpbin.org/delay/10') response = future.result() Exiting the ``with`` block calls ``close()``, which shuts down the session's own executor with ``cancel_futures=True``: every queued request that hasn't started running yet is cancelled immediately, not merely skipped one at a time. Here, with ``max_workers=1``, ``future`` is already done by the time the block exits, but neither ``future2`` nor ``future3`` has started, so both are cancelled together, saving the time and resources their requests would otherwise have used. A request that is already running when ``close()`` runs is left to finish. Using ProcessPoolExecutor ========================= A ``ProcessPoolExecutor`` can be supplied in place of a ``ThreadPoolExecutor`` to run requests in separate processes instead of threads, which is useful when per-request memory usage is high enough that cycling the interpreter is needed to release it back to the OS. Everything submitted to a process pool must be picklable, which means any ``FuturesSession`` subclass used this way must be importable at module scope (not defined inline, e.g. in a function or ``__main__`` script body). If something in the request isn't picklable, ``FuturesSession`` raises a ``RuntimeError`` pointing back to this documentation rather than letting the pickling failure surface on its own. .. code-block:: python from concurrent.futures import ProcessPoolExecutor from requests_futures.sessions import FuturesSession session = FuturesSession(executor=ProcessPoolExecutor(max_workers=10)) # ... use as before See the `full ProcessPoolExecutor guide`_ in the docs for a module-global callback example and more on the pickling requirements. Documentation ============= Full documentation, including everything below, lives at https://requests-futures.readthedocs.io/: * ``as_completed`` with a timeout * error handling across the future boundary (submit-time vs. ``result()``-time) * retries via a mounted ``HTTPAdapter`` and urllib3's ``Retry`` * sizing ``max_workers`` against the connection pool, including with a supplied ``session=`` * streaming responses (``stream=True``) and why it interacts badly with background work * sharing one executor across several sessions, and what ``close()`` does in that configuration * ``hooks``, the recommended replacement for the deprecated ``background_callback`` * the full ``ProcessPoolExecutor`` guide * the full thread-safety discussion Installation ============ pip install requests-futures .. _`requests`: https://github.com/kennethreitz/requests .. _`concurrent.futures`: http://docs.python.org/dev/library/concurrent.futures.html .. _`full thread-safety discussion`: https://requests-futures.readthedocs.io/en/latest/usage.html .. _`full ProcessPoolExecutor guide`: https://requests-futures.readthedocs.io/en/latest/usage.html ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5121133 requests_futures-1.1.0/requests_futures.egg-info/SOURCES.txt0000644000175000017500000000053615244425101022747 0ustar00rossrossLICENSE MANIFEST.in README.rst pyproject.toml requirements.txt requests_futures/__init__.py requests_futures/sessions.py requests_futures.egg-info/PKG-INFO requests_futures.egg-info/SOURCES.txt requests_futures.egg-info/dependency_links.txt requests_futures.egg-info/requires.txt requests_futures.egg-info/top_level.txt tests/test_requests_futures.py././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5065048 requests_futures-1.1.0/requests_futures.egg-info/dependency_links.txt0000644000175000017500000000000115244425101025125 0ustar00rossross ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5076892 requests_futures-1.1.0/requests_futures.egg-info/requires.txt0000644000175000017500000000050615244425101023460 0ustar00rossrossrequests>=1.2.0 [dev] pytest>=6.2.5 pytest-cov>=3.0.0 pytest-httpbin>=2.0.0 black<27.0.0,>=26.0.0 build>=0.7.0 changelet isort>=5.11.4 proviso pyflakes>=2.2.0 readme_renderer[rst]>=26.0 twine>=3.4.2 [docs] sphinx>=8.1.0 sphinx-copybutton>=0.5.2 sphinx_rtd_theme [test] pytest>=6.2.5 pytest-cov>=3.0.0 pytest-httpbin>=2.0.0 ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5078146 requests_futures-1.1.0/requests_futures.egg-info/top_level.txt0000644000175000017500000000002115244425101023602 0ustar00rossrossrequests_futures ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787960443.5899947 requests_futures-1.1.0/requirements.txt0000644000175000017500000000501715244416174017236 0ustar00rossross# DO NOT EDIT THIS FILE DIRECTLY - use ./script/update-requirements x-python-version-not-supported; python_version!='3.10' and python_version!='3.11' and python_version!='3.12' and python_version!='3.13' and python_version!='3.14' alabaster==1.0.0 anyio==4.14.2 babel==2.18.0 backports-tarfile==1.2.0; python_version=='3.10' or python_version=='3.11' black==26.5.1 blinker==1.9.0 brotlicffi==1.2.0.2 build==1.6.0 certifi==2026.7.22 cffi==2.1.1 changelet==0.6.1 charset-normalizer==3.5.1 click==8.5.0 coverage==7.15.4 cryptography==50.0.1 decorator==5.3.1 docutils==0.21.2; python_version=='3.10' docutils==0.22.4; python_version=='3.11' or python_version=='3.12' or python_version=='3.13' or python_version=='3.14' exceptiongroup==1.3.1; python_version=='3.10' flask==3.1.3 greenlet==2.0.2; python_version=='3.10' or python_version=='3.11' greenlet==3.5.5; python_version=='3.12' or python_version=='3.13' or python_version=='3.14' h11==0.16.0 hishel==1.3.1 httpbin==0.10.4 httpcore==1.0.9 httpx==0.28.1 id==1.6.1 idna==3.19 imagesize==2.0.1 importlib-metadata==9.0.0; python_version=='3.10' or python_version=='3.11' iniconfig==2.3.0 isort==9.0.1 itsdangerous==2.2.0 jaraco-classes==3.4.0 jaraco-context==6.1.2 jaraco-functools==4.6.0 jeepney==0.9.0 jinja2==3.1.6 keyring==25.7.0 markdown-it-py==4.2.0 markupsafe==3.0.3 mdurl==0.1.2 more-itertools==11.1.0 msgpack==1.2.2 mypy-extensions==1.1.0 nh3==0.3.7 packaging==26.3 pathspec==1.1.1 platformdirs==4.11.5 pluggy==1.6.0 proviso==0.3.0 pycparser==3.0 pyflakes==3.4.0 pygments==2.21.0 pyproject-hooks==1.2.0 pytest==9.1.1 pytest-cov==7.1.0 pytest-httpbin==2.1.0 pytokens==0.4.1 pyyaml==6.0.3 readme-renderer==45.0 requests==2.34.2 requests-toolbelt==1.0.0 resolvelib==1.2.1 rfc3986==2.0.0 rich==15.0.0 roman-numerals==4.1.0; python_version=='3.11' or python_version=='3.12' or python_version=='3.13' or python_version=='3.14' secretstorage==3.5.0 semver==3.0.4 setuptools==84.0.0 six==1.17.0 snowballstemmer==3.1.1 sphinx==8.1.3; python_version=='3.10' sphinx==9.0.4; python_version=='3.11' sphinx==9.1.0; python_version=='3.12' or python_version=='3.13' or python_version=='3.14' sphinx-copybutton==0.5.2 sphinx-rtd-theme==3.1.0 sphinxcontrib-applehelp==2.0.0 sphinxcontrib-devhelp==2.0.0 sphinxcontrib-htmlhelp==2.1.0 sphinxcontrib-jquery==4.1 sphinxcontrib-jsmath==1.0.1 sphinxcontrib-qthelp==2.0.0 sphinxcontrib-serializinghtml==2.0.0 tomli==2.4.1; python_version=='3.10' twine==7.0.0 typing-extensions==4.16.0 unearth==0.18.3 urllib3==2.7.0 werkzeug==3.1.8 zipp==4.1.0; python_version=='3.10' or python_version=='3.11' ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5176518 requests_futures-1.1.0/setup.cfg0000644000175000017500000000004615244425101015556 0ustar00rossross[egg_info] tag_build = tag_date = 0 ././@PaxHeader0000000000000000000000000000003400000000000010212 xustar0028 mtime=1787963968.5160701 requests_futures-1.1.0/tests/0000755000175000017500000000000015244425101015077 5ustar00rossross././@PaxHeader0000000000000000000000000000003300000000000010211 xustar0027 mtime=1787960443.590479 requests_futures-1.1.0/tests/test_requests_futures.py0000644000175000017500000006645215244416174022167 0ustar00rossross#!/usr/bin/env python # -*- coding: utf-8 -*- """Tests for Requests.""" import logging from concurrent.futures import Future, ProcessPoolExecutor from os.path import basename from threading import Event, Thread from time import monotonic, sleep from unittest import TestCase, main import pytest from requests import Response, Session, session from requests.adapters import ( DEFAULT_POOLSIZE, DEFAULT_RETRIES, BaseAdapter, HTTPAdapter, Retry, ) from requests_futures.sessions import PICKLE_ERROR, FuturesSession logging.basicConfig(level=logging.DEBUG) logging.getLogger('urllib3.connectionpool').level = logging.WARNING @pytest.fixture(scope="class", autouse=True) def httpbin_on_class(request, httpbin): request.cls.httpbin = httpbin class RequestsTestCase(TestCase): def make_session(self, *args, **kwargs): """Builds a FuturesSession and registers it to be closed at the end of the test, so tests don't leak executor threads.""" sess = FuturesSession(*args, **kwargs) self.addCleanup(sess.close) return sess def test_futures_session(self): # basic futures get sess = self.make_session() future = sess.get(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) # non-200, 404 future = sess.get(self.httpbin.join('status/404')) resp = future.result() self.assertEqual(404, resp.status_code) def cb(s, r): self.assertIsInstance(s, FuturesSession) self.assertIsInstance(r, Response) # add the parsed json data to the response r.data = r.json() future = sess.get(self.httpbin.join('get'), background_callback=cb) # this should block until complete resp = future.result() self.assertEqual(200, resp.status_code) # make sure the callback was invoked self.assertTrue(hasattr(resp, 'data')) def rasing_cb(s, r): raise Exception('boom') future = sess.get( self.httpbin.join('get'), background_callback=rasing_cb ) with self.assertRaises(Exception) as cm: resp = future.result() self.assertEqual('boom', cm.exception.args[0]) def test_background_callback_falsy_return(self): """A background_callback's falsy return value must not be discarded in favor of the Response.""" sess = self.make_session() def cb(s, r): return {} future = sess.get(self.httpbin.join('get'), background_callback=cb) self.assertEqual({}, future.result()) def test_background_callback_deprecated(self): """A background_callback param must emit a DeprecationWarning attributed to the caller, not to sessions.py.""" sess = self.make_session() def cb(s, r): pass with self.assertWarns(DeprecationWarning) as cm: future = sess.get(self.httpbin.join('get'), background_callback=cb) self.assertIn('hooks', str(cm.warning)) # pins stacklevel: the warning points at this file, not sessions.py self.assertEqual(basename(__file__), basename(cm.filename)) future.result() def test_hooks_response(self): """A `hooks={'response': fn}` hook runs against the Response, and unlike `background_callback` never replaces what the future resolves to: hook return values are requests' own business.""" sess = self.make_session() def hook(response, *args, **kwargs): response.hooked = True future = sess.get(self.httpbin.join('get'), hooks={'response': hook}) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertTrue(resp.hooked) def test_options(self): sess = self.make_session() future = sess.options(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) def test_head(self): sess = self.make_session() future = sess.head(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) def test_post(self): sess = self.make_session() future = sess.post(self.httpbin.join('post'), data={'a': 'b'}) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertEqual({'a': 'b'}, resp.json()['form']) future = sess.post(self.httpbin.join('post'), json={'a': 'b'}) resp = future.result() self.assertEqual(200, resp.status_code) self.assertEqual({'a': 'b'}, resp.json()['json']) def test_put(self): sess = self.make_session() future = sess.put(self.httpbin.join('put'), data={'a': 'b'}) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertEqual({'a': 'b'}, resp.json()['form']) def test_patch(self): sess = self.make_session() future = sess.patch(self.httpbin.join('patch'), data={'a': 'b'}) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertEqual({'a': 'b'}, resp.json()['form']) def test_delete(self): sess = self.make_session() future = sess.delete(self.httpbin.join('delete')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) def test_supplied_session(self): """Tests the `session` keyword argument.""" requests_session = session() requests_session.headers['Foo'] = 'bar' sess = self.make_session(session=requests_session) future = sess.get(self.httpbin.join('headers')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertEqual(resp.json()['headers']['Foo'], 'bar') def test_max_workers(self): """Tests the `max_workers` shortcut.""" from concurrent.futures import ThreadPoolExecutor session = self.make_session() self.assertEqual(session.executor._max_workers, 8) session = self.make_session(max_workers=5) self.assertEqual(session.executor._max_workers, 5) executor = ThreadPoolExecutor(max_workers=10) self.addCleanup(executor.shutdown) session = self.make_session(executor=executor) self.assertEqual(session.executor._max_workers, 10) session = self.make_session(executor=executor, max_workers=5) self.assertEqual(session.executor._max_workers, 10) def test_adapter_kwargs(self): """Tests the `adapter_kwargs` shortcut.""" from concurrent.futures import ThreadPoolExecutor session = self.make_session() self.assertFalse(session.get_adapter('http://')._pool_block) session = self.make_session( max_workers=DEFAULT_POOLSIZE + 1, adapter_kwargs={'pool_block': True}, ) adapter = session.get_adapter('http://') self.assertTrue(adapter._pool_block) self.assertEqual(adapter._pool_connections, DEFAULT_POOLSIZE + 1) self.assertEqual(adapter._pool_maxsize, DEFAULT_POOLSIZE + 1) executor = ThreadPoolExecutor(max_workers=10) self.addCleanup(executor.shutdown) session = self.make_session( executor=executor, adapter_kwargs={'pool_connections': 20} ) self.assertEqual(session.get_adapter('http://')._pool_connections, 20) def test_adapter_kwargs_with_supplied_session(self): """`adapter_kwargs`/`max_workers` sizing must apply to the supplied `session`, since that's what actually serves requests.""" requests_session = session() futures_session = self.make_session( session=requests_session, max_workers=DEFAULT_POOLSIZE + 1 ) adapter = requests_session.get_adapter('http://') self.assertEqual(adapter._pool_connections, DEFAULT_POOLSIZE + 1) self.assertEqual(adapter._pool_maxsize, DEFAULT_POOLSIZE + 1) # the FuturesSession itself should be left with its defaults self.assertEqual( futures_session.get_adapter('http://')._pool_maxsize, DEFAULT_POOLSIZE, ) requests_session = session() self.make_session( session=requests_session, adapter_kwargs={'pool_block': True} ) self.assertTrue(requests_session.get_adapter('http://')._pool_block) def test_supplied_session_adapter_preserved(self): """Applying pool sizing to a supplied session must reconfigure its existing adapters in place rather than replace them, so retry policy and custom adapter behavior survive.""" requests_session = session() retrying_adapter = HTTPAdapter( max_retries=Retry(total=5), pool_block=True ) requests_session.mount('https://', retrying_adapter) self.make_session( session=requests_session, max_workers=DEFAULT_POOLSIZE + 1 ) adapter = requests_session.get_adapter('https://') self.assertIs(adapter, retrying_adapter) self.assertEqual(adapter.max_retries.total, 5) self.assertTrue(adapter._pool_block) self.assertEqual(adapter._pool_connections, DEFAULT_POOLSIZE + 1) self.assertEqual(adapter._pool_maxsize, DEFAULT_POOLSIZE + 1) self.assertEqual( adapter.poolmanager.connection_pool_kw['maxsize'], DEFAULT_POOLSIZE + 1, ) # a custom adapter subclass keeps its class and its own attributes class CustomAdapter(HTTPAdapter): def __init__(self, marker, **kwargs): self.marker = marker super(CustomAdapter, self).__init__(**kwargs) requests_session = session() custom_adapter = CustomAdapter('mine') requests_session.mount('https://', custom_adapter) self.make_session( session=requests_session, max_workers=DEFAULT_POOLSIZE + 1 ) adapter = requests_session.get_adapter('https://') self.assertIs(adapter, custom_adapter) self.assertIsInstance(adapter, CustomAdapter) self.assertEqual(adapter.marker, 'mine') self.assertEqual(adapter._pool_maxsize, DEFAULT_POOLSIZE + 1) # adapter_kwargs' max_retries is applied the same way requests_session = session() requests_session.mount('https://', HTTPAdapter()) self.make_session( session=requests_session, adapter_kwargs={'max_retries': 3} ) self.assertEqual( requests_session.get_adapter('https://').max_retries.total, 3 ) # max_retries=DEFAULT_RETRIES matches HTTPAdapter's own handling # (a bare Retry(0, read=False) rather than Retry.from_int) requests_session = session() requests_session.mount('https://', HTTPAdapter()) self.make_session( session=requests_session, adapter_kwargs={'max_retries': DEFAULT_RETRIES}, ) self.assertEqual( requests_session.get_adapter('https://').max_retries.total, 0 ) # a non-HTTPAdapter is left alone rather than blowing up class MockAdapter(BaseAdapter): def send(self, *args, **kwargs): pass def close(self): pass requests_session = session() mock_adapter = MockAdapter() requests_session.mount('http://', mock_adapter) self.make_session( session=requests_session, max_workers=DEFAULT_POOLSIZE + 1 ) self.assertIs(requests_session.get_adapter('http://'), mock_adapter) def test_supplied_session_proxy_manager_resized(self): """A supplied session's cached proxy managers must pick up new pool sizing too, since `init_poolmanager` alone doesn't touch them.""" requests_session = session() adapter = requests_session.get_adapter('http://') stale_proxy_manager = adapter.proxy_manager_for('http://proxy.example') self.assertEqual( stale_proxy_manager.connection_pool_kw['maxsize'], DEFAULT_POOLSIZE ) self.make_session( session=requests_session, max_workers=DEFAULT_POOLSIZE + 1, adapter_kwargs={'pool_block': True}, ) # same adapter, but the proxy manager was rebuilt at the new size self.assertIs(requests_session.get_adapter('http://'), adapter) proxy_manager = adapter.proxy_manager_for('http://proxy.example') self.assertIsNot(proxy_manager, stale_proxy_manager) self.assertEqual( proxy_manager.connection_pool_kw['maxsize'], DEFAULT_POOLSIZE + 1 ) self.assertTrue(proxy_manager.connection_pool_kw['block']) # when pool settings are unchanged, the cached proxy manager (and # its warm connections) is left alone requests_session = session() adapter = requests_session.get_adapter('http://') cached_proxy_manager = adapter.proxy_manager_for('http://proxy.example') self.make_session( session=requests_session, adapter_kwargs={'max_retries': 3} ) self.assertIs( adapter.proxy_manager_for('http://proxy.example'), cached_proxy_manager, ) self.assertEqual(adapter.max_retries.total, 3) def test_redirect(self): """Tests for the ability to cleanly handle redirects.""" sess = self.make_session() future = sess.get(self.httpbin.join('redirect-to?url=get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) future = sess.get(self.httpbin.join('redirect-to?url=status/404')) resp = future.result() self.assertEqual(404, resp.status_code) def test_context(self): class FuturesSessionTestHelper(FuturesSession): def __init__(self, *args, **kwargs): super(FuturesSessionTestHelper, self).__init__(*args, **kwargs) self._exit_called = False def __exit__(self, *args, **kwargs): self._exit_called = True return super(FuturesSessionTestHelper, self).__exit__( *args, **kwargs ) passout = None with FuturesSessionTestHelper() as sess: passout = sess future = sess.get(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertTrue(passout._exit_called) def test_close_cancels_queued_requests(self): request_started = Event() finish_request = Event() request_finished = Event() adapter_closed = Event() adapter_closed_after_request = Event() session = FuturesSession(max_workers=1) class TrackingAdapter(BaseAdapter): def send(self, *args, **kwargs): pass def close(self): if request_finished.is_set(): adapter_closed_after_request.set() adapter_closed.set() session.mount('test://', TrackingAdapter()) def request(): request_started.set() finish_request.wait() request_finished.set() running = session.executor.submit(request) self.assertTrue(request_started.wait(timeout=1)) queued = session.executor.submit(lambda: None) close_thread = Thread(target=session.close) close_thread.start() try: deadline = monotonic() + 1 while not queued.done() and monotonic() < deadline: sleep(0.01) self.assertTrue(queued.cancelled()) finally: finish_request.set() close_thread.join(timeout=1) self.assertFalse(close_thread.is_alive()) running.result() self.assertTrue(adapter_closed.is_set()) self.assertTrue(adapter_closed_after_request.is_set()) def test_close_does_not_close_supplied_session(self): class TrackingSession(Session): def __init__(self): super(TrackingSession, self).__init__() self.close_calls = 0 def close(self): self.close_calls += 1 super(TrackingSession, self).close() requests_session = TrackingSession() futures_session = FuturesSession(session=requests_session) futures_session.close() self.assertEqual(requests_session.close_calls, 0) requests_session.close() self.assertEqual(requests_session.close_calls, 1) def test_close_waits_for_requests_with_supplied_executor(self): from concurrent.futures import ThreadPoolExecutor request_started = Event() finish_request = Event() request_finished = Event() adapter_closed = Event() adapter_closed_after_request = Event() executor = ThreadPoolExecutor(max_workers=1) session = FuturesSession(executor=executor) close_threads = [] class TrackingAdapter(BaseAdapter): def send(self, request, **kwargs): request_started.set() finish_request.wait() request_finished.set() response = Response() response.request = request response.status_code = 200 response.url = request.url return response def close(self): if request_finished.is_set(): adapter_closed_after_request.set() adapter_closed.set() try: session.mount('test://', TrackingAdapter()) running = session.get('test://running') self.assertTrue(request_started.wait(timeout=1)) queued = session.get('test://queued') unrelated = executor.submit(lambda: 'unrelated') close_threads = [Thread(target=session.close) for _ in range(2)] for close_thread in close_threads: close_thread.start() deadline = monotonic() + 1 while not queued.done() and monotonic() < deadline: sleep(0.01) self.assertTrue(queued.cancelled()) self.assertTrue( all(close_thread.is_alive() for close_thread in close_threads) ) with self.assertRaisesRegex(RuntimeError, 'after close'): session.get('test://closing') finish_request.set() for close_thread in close_threads: close_thread.join(timeout=1) self.assertTrue( all( not close_thread.is_alive() for close_thread in close_threads ) ) self.assertEqual(running.result().status_code, 200) self.assertEqual(unrelated.result(), 'unrelated') self.assertTrue(adapter_closed.is_set()) self.assertTrue(adapter_closed_after_request.is_set()) with self.assertRaisesRegex(RuntimeError, 'after close'): session.get('test://closed') self.assertTrue(executor.submit(lambda: True).result()) finally: finish_request.set() for close_thread in close_threads: close_thread.join(timeout=1) executor.shutdown() # << test process pool executor >> # see discussion https://github.com/ross/requests-futures/issues/11 def global_cb_modify_response(s, r): """add the parsed json data to the response""" assert s, FuturesSession assert r, Response r.data = r.json() # reassign rather than .append(): __attrs__ is a class attribute # shared by every Response, and workers are reused across requests r.__attrs__ = r.__attrs__ + ['data'] def global_cb_return_result(s, r): """simply return parsed json data""" assert s, FuturesSession assert r, Response return r.json() def global_rasing_cb(s, r): raise Exception('boom') def global_hook_mark_response(response, *args, **kwargs): """A `hooks={'response': fn}` callable, module-global because anything submitted to a ProcessPoolExecutor must be picklable.""" response.hooked = True # reassign rather than .append(): __attrs__ is a class attribute # shared by every Response, and workers are reused across requests response.__attrs__ = response.__attrs__ + ['hooked'] class RequestsProcessPoolTestCase(TestCase): def setUp(self): self.proc_executor = ProcessPoolExecutor(max_workers=2) self.addCleanup(self.proc_executor.shutdown) self.session = session() def test_futures_session(self): self._assert_futures_session() def test_futures_existing_session(self): self.session.headers['Foo'] = 'bar' self._assert_futures_session(session=self.session) def test_hooks_response(self): """A `hooks={'response': fn}` hook runs in the worker process and the future still resolves to the (mutated) Response.""" sess = FuturesSession(executor=self.proc_executor, session=self.session) future = sess.get( self.httpbin.join('get'), hooks={'response': global_hook_mark_response}, ) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertTrue(resp.hooked) def _assert_futures_session(self, session=None): # basic futures get if session: sess = FuturesSession(executor=self.proc_executor, session=session) else: sess = FuturesSession(executor=self.proc_executor) future = sess.get(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) # non-200, 404 future = sess.get(self.httpbin.join('status/404')) resp = future.result() self.assertEqual(404, resp.status_code) future = sess.get( self.httpbin.join('get'), background_callback=global_cb_modify_response, ) # this should block until complete resp = future.result() if session: self.assertEqual(resp.json()['headers']['Foo'], 'bar') self.assertEqual(200, resp.status_code) # make sure the callback was invoked self.assertTrue(hasattr(resp, 'data')) future = sess.get( self.httpbin.join('get'), background_callback=global_cb_return_result, ) # this should block until complete resp = future.result() # make sure the callback was invoked self.assertIsInstance(resp, dict) future = sess.get( self.httpbin.join('get'), background_callback=global_rasing_cb ) with self.assertRaises(Exception) as cm: resp = future.result() self.assertEqual('boom', cm.exception.args[0]) # Tests for the ability to cleanly handle redirects future = sess.get(self.httpbin.join('redirect-to?url=get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) future = sess.get(self.httpbin.join('redirect-to?url=status/404')) resp = future.result() self.assertEqual(404, resp.status_code) def test_unpicklable_hooks_raises(self): """A `hooks` callable that can't be pickled must raise RuntimeError at submit time, not a raw pickling error out of the Future.""" def local_hook(r, *args, **kwargs): return r sess = FuturesSession(executor=self.proc_executor, session=self.session) with self.assertRaises(RuntimeError) as cm: sess.get(self.httpbin.join('get'), hooks={'response': local_hook}) self.assertEqual(PICKLE_ERROR, str(cm.exception)) self.assertIsNotNone(cm.exception.__cause__) def test_unpicklable_data_raises(self): """An unpicklable request argument (e.g. a file-like `data=`) must raise RuntimeError at submit time.""" sess = FuturesSession(executor=self.proc_executor, session=self.session) f = open(__file__) try: with self.assertRaises(RuntimeError) as cm: sess.post(self.httpbin.join('post'), data=f) self.assertEqual(PICKLE_ERROR, str(cm.exception)) self.assertIsNotNone(cm.exception.__cause__) finally: f.close() def test_unpicklable_background_callback_raises(self): """A local-function `background_callback` must raise RuntimeError at submit time.""" def local_cb(s, r): return r sess = FuturesSession(executor=self.proc_executor, session=self.session) with self.assertRaises(RuntimeError) as cm: sess.get(self.httpbin.join('get'), background_callback=local_cb) self.assertEqual(PICKLE_ERROR, str(cm.exception)) self.assertIsNotNone(cm.exception.__cause__) def test_picklable_request_args_submitted(self): """Ordinary picklable kwargs must still go through the widened pickle check.""" sess = FuturesSession(executor=self.proc_executor, session=self.session) future = sess.get(self.httpbin.join('get'), params={'a': 'b'}) resp = future.result() self.assertEqual(200, resp.status_code) def test_context(self): self._assert_context() def test_context_with_session(self): self._assert_context(session=self.session) def _assert_context(self, session=None): if session: helper_instance = TopLevelContextHelper( executor=self.proc_executor, session=self.session ) else: helper_instance = TopLevelContextHelper(executor=self.proc_executor) passout = None with helper_instance as sess: passout = sess future = sess.get(self.httpbin.join('get')) self.assertIsInstance(future, Future) resp = future.result() self.assertIsInstance(resp, Response) self.assertEqual(200, resp.status_code) self.assertTrue(passout._exit_called) class TopLevelContextHelper(FuturesSession): def __init__(self, *args, **kwargs): super(TopLevelContextHelper, self).__init__(*args, **kwargs) self._exit_called = False def __exit__(self, *args, **kwargs): self._exit_called = True return super(TopLevelContextHelper, self).__exit__(*args, **kwargs) if __name__ == '__main__': main()