pax_global_header00006660000000000000000000000064150577767600014536gustar00rootroot0000000000000052 comment=19a86cc86b58f8f58799a449e5d1d7b7e92f40d0 KenanHanke-rbloom-b215628/000077500000000000000000000000001505777676000153045ustar00rootroot00000000000000KenanHanke-rbloom-b215628/.github/000077500000000000000000000000001505777676000166445ustar00rootroot00000000000000KenanHanke-rbloom-b215628/.github/workflows/000077500000000000000000000000001505777676000207015ustar00rootroot00000000000000KenanHanke-rbloom-b215628/.github/workflows/CI.yml000066400000000000000000000066511505777676000217270ustar00rootroot00000000000000# This file is autogenerated by maturin v1.5.1 # To update, run # # maturin generate-ci github -o .github/workflows/CI.yml # name: CI on: push: branches: - main - master tags: - "*" pull_request: workflow_dispatch: permissions: contents: read jobs: linux: runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - runner: ubuntu-latest target: x86_64 - runner: ubuntu-latest target: x86 - runner: ubuntu-latest target: aarch64 - runner: ubuntu-latest target: armv7 - runner: ubuntu-latest target: s390x - runner: ubuntu-latest target: ppc64le steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out dist sccache: "true" manylinux: auto - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-linux-${{ matrix.platform.target }} path: dist windows: runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - runner: windows-latest target: x64 - runner: windows-latest target: x86 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.10" architecture: ${{ matrix.platform.target }} - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out dist sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-windows-${{ matrix.platform.target }} path: dist macos: runs-on: ${{ matrix.platform.runner }} strategy: matrix: platform: - runner: macos-latest target: x86_64 - runner: macos-14 target: aarch64 steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.10" - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out dist sccache: "true" - name: Upload wheels uses: actions/upload-artifact@v4 with: name: wheels-macos-${{ matrix.platform.target }} path: dist sdist: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Build sdist uses: PyO3/maturin-action@v1 with: command: sdist args: --out dist - name: Upload sdist uses: actions/upload-artifact@v4 with: name: wheels-sdist path: dist release: name: Release runs-on: ubuntu-latest if: "startsWith(github.ref, 'refs/tags/')" needs: [linux, windows, macos, sdist] steps: - uses: actions/download-artifact@v4 - name: Publish to PyPI uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} with: command: upload args: --non-interactive --skip-existing wheels-*/* KenanHanke-rbloom-b215628/.gitignore000066400000000000000000000012561505777676000173000ustar00rootroot00000000000000/target # Byte-compiled / optimized / DLL files __pycache__/ .pytest_cache/ *.py[cod] # C extensions *.so # Distribution / packaging .Python .venv/ env/ bin/ build/ develop-eggs/ dist/ eggs/ lib/ lib64/ parts/ sdist/ var/ include/ man/ venv/ *.egg-info/ .installed.cfg *.egg # Installer logs pip-log.txt pip-delete-this-directory.txt pip-selfcheck.json # Unit test / coverage reports htmlcov/ .tox/ .coverage .cache nosetests.xml coverage.xml # Translations *.mo # Mr Developer .mr.developer.cfg .project .pydevproject # Rope .ropeproject # Django stuff: *.log *.pot .DS_Store # Sphinx documentation docs/_build/ # PyCharm .idea/ # VSCode .vscode/ # Pyenv .python-version KenanHanke-rbloom-b215628/Cargo.lock000066400000000000000000000102151505777676000172100ustar00rootroot00000000000000# This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 4 [[package]] name = "autocfg" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "heck" version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "indoc" version = "2.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" [[package]] name = "libc" version = "0.2.175" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a82ae493e598baaea5209805c49bbf2ea7de956d50d7da0da1164f9c6d28543" [[package]] name = "memoffset" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" dependencies = [ "autocfg", ] [[package]] name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "portable-atomic" version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "proc-macro2" version = "1.0.101" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" dependencies = [ "unicode-ident", ] [[package]] name = "pyo3" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ba0117f4212101ee6544044dae45abe1083d30ce7b29c4b5cbdfa2354e07383" dependencies = [ "indoc", "libc", "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", "unindent", ] [[package]] name = "pyo3-build-config" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fc6ddaf24947d12a9aa31ac65431fb1b851b8f4365426e182901eabfb87df5f" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "025474d3928738efb38ac36d4744a74a400c901c7596199e20e45d98eb194105" dependencies = [ "libc", "pyo3-build-config", ] [[package]] name = "pyo3-macros" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2e64eb489f22fe1c95911b77c44cc41e7c19f3082fc81cce90f657cdc42ffded" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", "syn", ] [[package]] name = "pyo3-macros-backend" version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "100246c0ecf400b475341b8455a9213344569af29a3c841d29270e53102e0fcf" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", "syn", ] [[package]] name = "quote" version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" dependencies = [ "proc-macro2", ] [[package]] name = "rbloom" version = "1.5.4" dependencies = [ "pyo3", ] [[package]] name = "syn" version = "2.0.106" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "target-lexicon" version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "unicode-ident" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" [[package]] name = "unindent" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" KenanHanke-rbloom-b215628/Cargo.toml000066400000000000000000000005371505777676000172410ustar00rootroot00000000000000[package] name = "rbloom" version = "1.5.4" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] name = "rbloom" crate-type = ["cdylib"] [dependencies] pyo3 = { version = "0.26", features = [ "extension-module", "abi3-py37", ] } # stable ABI with minimum Python version 3.7 KenanHanke-rbloom-b215628/LICENSE000066400000000000000000000022301505777676000163060ustar00rootroot00000000000000MIT License Copyright (c) 2023-2025 Kenan Hanke Copyright (c) 2023-2025 Zachary Dremann Copyright (c) 2023 Rory McNamara Copyright (c) 2024 Dan Lenski Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. KenanHanke-rbloom-b215628/README.md000066400000000000000000000267261505777676000166000ustar00rootroot00000000000000# rBloom [![PyPI](https://img.shields.io/pypi/v/rbloom)](https://pypi.org/project/rbloom/) [![license](https://img.shields.io/github/license/KenanHanke/rbloom)](https://github.com/KenanHanke/rbloom/blob/main/LICENSE) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.17084810.svg)](https://doi.org/10.5281/zenodo.17084810) [![build](https://img.shields.io/github/actions/workflow/status/KenanHanke/rbloom/CI.yml)](https://github.com/KenanHanke/rbloom/actions) A fast, simple and lightweight [Bloom filter](https://en.wikipedia.org/wiki/Bloom_filter) library for Python, implemented in Rust. It's designed to be as pythonic as possible, mimicking the built-in `set` type where it can, and works with any hashable object. While it's a new library (this project was started in 2023), it's currently the fastest option for Python by a long shot (see the section [Benchmarks](#benchmarks)). Releases are published on [PyPI](https://pypi.org/project/rbloom/). ## Quickstart This library defines only one class, which can be used as follows: ```python >>> from rbloom import Bloom >>> bf = Bloom(200, 0.01) # 200 items max, false positive rate of 1% >>> bf.add("hello") >>> "hello" in bf True >>> "world" in bf False >>> bf.update(["hello", "world"]) # "hello" and "world" now in bf >>> other_bf = Bloom(200, 0.01) ### add some items to other_bf >>> third_bf = bf | other_bf # third_bf now contains all items in # bf and other_bf >>> third_bf = bf.copy() ... third_bf.update(other_bf) # same as above >>> bf.issubset(third_bf) # bf <= third_bf also works True ``` For the full API, see the section [Documentation](#documentation). ## Installation On almost all platforms, simply run: ```sh pip install rbloom ``` If you're on an uncommon platform, this may cause pip to build the library from source, which requires the Rust [toolchain](https://www.rust-lang.org/tools/install). You can also build `rbloom` by cloning this repository and running [maturin](https://github.com/PyO3/maturin): ```sh maturin build --release ``` This will create a wheel in the `target/wheels/` directory, which can subsequently also be passed to pip. ## Why rBloom? Why should you use this library instead of one of the other Bloom filter libraries on PyPI? - **Simple:** Almost all important methods work exactly like their counterparts in the built-in `set` [type](https://docs.python.org/3/library/stdtypes.html#set-types-set-frozenset). - **Fast:** `rbloom` is implemented in Rust, which makes it blazingly fast. See section [Benchmarks](#benchmarks) for more information. - **Lightweight:** `rbloom` has no dependencies of its own. - **Maintainable:** This library is very concise, and it's written in idiomatic Rust. Even if I were to stop maintaining `rbloom` (which I don't intend to), it would be trivially easy for you to fork it and keep it working for you. I started `rbloom` because I was looking for a simple Bloom filter dependency for a project, and the only sufficiently fast option (`pybloomfiltermmap3`) was segfaulting on recent Python versions. `rbloom` ended up being twice as fast and has grown to encompass a more complete API (e.g. with set comparisons like `issubset`). Do note that it doesn't use mmapped files, however. This shouldn't be an issue in most cases, as the random access heavy nature of a Bloom filter negates the benefits of mmap after very few operations, but it is something to keep in mind for edge cases. ## Benchmarks The following simple benchmark was implemented in the respective API of each library (see the [comparison benchmarks](benchmarks/compare.py)): ```python bf = Bloom(10_000_000, 0.01) for i in range(10_000_000): bf.add(i + 0.5) # floats because ints are hashed as themselves for i in range(10_000_000): assert i + 0.5 in bf ``` This resulted in the following average runtimes on an M1 Pro (confirmed to be proportional to runtimes on an Intel machine): | Library | Time | Notes | | ------------------------------------------------------------------ | ------- | ------------------------------------- | | [rBloom](https://pypi.org/project/rbloom/) | 2.52s | works out-of-the-box | | [pybloomfiltermmap3](https://pypi.org/project/pybloomfiltermmap3/) | 4.78s | unreliable [1] | | [pybloom3](https://pypi.org/project/pybloom3/) | 46.76s | works out-of-the-box | | [Flor](https://pypi.org/project/Flor/) | 76.94s | doesn't work on arbitrary objects [2] | | [bloom-filter2](https://pypi.org/project/bloom-filter2/) | 165.54s | doesn't work on arbitrary objects [2] | [1] The official package failed to install on Python 3.11 and kept segfaulting on 3.10 (Linux, January 2023). It seems to be fine for now (October 2024). [2] I was forced to convert to a byte representation, which is bad default behavior as it presents the problems mentioned below in the section "Cryptographic security". Also note that `rbloom` is compiled against a stable ABI for portability, and that you can get a small but measurable speedup by removing the `"abi3-py37"` flag from `Cargo.toml` and building it yourself. ## Documentation This library defines only one class, the signature of which should be thought of as follows. Note that only the first few methods differ from the built-in `set` type: ```python class Bloom: # expected_items: max number of items to be added to the filter # false_positive_rate: max false positive rate of the filter # hash_func: optional argument, see section "Cryptographic security" def __init__(self, expected_items: int, false_positive_rate: float, hash_func=__builtins__.hash) @property def size_in_bits(self) -> int # number of buckets in the filter @property def hash_func(self) -> Callable[[Any], int] # retrieve the hash_func # given to __init__ @property def approx_items(self) -> float # estimated number of items in # the filter # see section "Persistence" for more information on these four methods @classmethod def load(cls, filepath: str, hash_func) -> Bloom def save(self, filepath: str) @classmethod def load_bytes(cls, data: bytes, hash_func) -> Bloom def save_bytes(self) -> bytes ##################################################################### # ALL SUBSEQUENT METHODS ARE # # EQUIVALENT TO THE CORRESPONDING METHODS # # OF THE BUILT-IN SET TYPE # ##################################################################### def add(self, obj) # add obj to self def __contains__(self, obj) -> bool # check if obj in self def __bool__(self) -> bool # False if empty def __repr__(self) -> str # basic info def __or__(self, other: Bloom) -> Bloom # self | other def __ior__(self, other: Bloom) # self |= other def __and__(self, other: Bloom) -> Bloom # self & other def __iand__(self, other: Bloom) # self &= other # these extend the functionality of __or__, __ior__, __and__, __iand__ def union(self, *others: Union[Iterable, Bloom]) -> Bloom # __or__ def update(self, *others: Union[Iterable, Bloom]) # __ior__ def intersection(self, *others: Union[Iterable, Bloom]) -> Bloom # __and__ def intersection_update(self, *others: Union[Iterable, Bloom]) # __iand__ # these implement <, >, <=, >=, ==, != def __lt__, __gt__, __le__, __ge__, __eq__, __ne__(self, other: Bloom) -> bool def issubset(self, other: Bloom) -> bool # self <= other def issuperset(self, other: Bloom) -> bool # self >= other def clear(self) # remove all items def copy(self) -> Bloom # duplicate self ``` To prevent death and destruction, the bitwise set operations only work on filters where all parameters are equal (including the hash functions being the exact same object). Because this is a Bloom filter, the `__contains__` and `approx_items` methods are probabilistic, as are all the methods that compare two filters (such as `__le__` and `issubset`). ## Cryptographic security Python's built-in hash function is designed to be fast, not maximally collision-resistant, so if your program depends on the false positive rate being perfectly correct, you may want to supply your own hash function. This is especially the case when working with very large filters (more than a few tens of millions of items) or when false positives are very costly and could be exploited by an adversary. Just make sure that your hash function returns an integer between -2^127 and 2^127 - 1. Feel free to use the following example in your own code: ```python from rbloom import Bloom from hashlib import sha256 from pickle import dumps def hash_func(obj): h = sha256(dumps(obj)).digest() # use sys.byteorder instead of "big" for a small speedup when # reproducibility across machines isn't a concern return int.from_bytes(h[:16], "big", signed=True) bf = Bloom(100_000_000, 0.01, hash_func) ``` When you throw away Python's built-in hash function and start hashing serialized representations of objects, however, you open up a breach into the scary realm of the unpythonic: - Numbers like `1`, `1.0`, `1 + 0j` and `True` will suddenly no longer be equal. - Instances of classes with custom hashing logic (e.g. to stop caches inside instances from affecting their hashes) will suddenly display undefined behavior. - Objects that can't be serialized simply won't be hashable at all. Making you supply your own hash function in this case is a deliberate design decision intended to show you what you're doing and prevent you from shooting yourself in the foot. Also note that using a custom hash will incur a performance penalty over using the built-in hash. ## Persistence The `save` and `load` methods, along with their byte-oriented counterparts `save_bytes` and `load_bytes`, allow you to save and load filters to and from disk/Python `bytes` objects. However, as the built-in hash function's salt changes between invocations of Python, they only work on filters with custom hash functions. Note that it is your responsibility to ensure that the hash function you supply to the loading functions is the same as the one originally used by the filter you're loading! ```python bf = Bloom(10_000, 0.01, some_hash_func) bf.add("hello") bf.add("world") # saving to a file bf.save("bf.bloom") # loading from a file loaded_bf = Bloom.load("bf.bloom", some_hash_func) assert loaded_bf == bf # saving to bytes bf_bytes = bf.save_bytes() # loading from bytes loaded_bf_from_bytes = Bloom.load_bytes(bf_bytes, some_hash_func) assert loaded_bf_from_bytes == bf ``` The size of the file is `bf.size_in_bits / 8 + 8` bytes. --- **Statement of attribution:** Bloom filters were originally proposed in [(Bloom, 1970)](https://doi.org/10.1145/362686.362692). Furthermore, this implementation makes use of a constant recommended by [(L'Ecuyer, 1999)](https://doi.org/10.1090/S0025-5718-99-00996-5) for redistributing the entropy of a single hash over multiple integers using a linear congruential generator. KenanHanke-rbloom-b215628/benchmarks/000077500000000000000000000000001505777676000174215ustar00rootroot00000000000000KenanHanke-rbloom-b215628/benchmarks/compare.py000066400000000000000000000025131505777676000214220ustar00rootroot00000000000000import struct import timeit import pybloomfilter import bloom_filter2 import flor import pybloom import rbloom NUM_ITEMS = 10_000_000 def run(ty): bf = ty(NUM_ITEMS, 0.01) for i in range(NUM_ITEMS): bf.add(i + 0.5) # floats because ints are hashed as themselves for i in range(NUM_ITEMS): if i + 0.5 not in bf: raise ValueError("Should be no false negatives") def run_bytes(ty): bf = ty(NUM_ITEMS, 0.01) for i in range(NUM_ITEMS): bf.add(struct.pack("d", i + 0.5)) for i in range(NUM_ITEMS): if struct.pack("d", i + 0.5) not in bf: raise ValueError("Should be no false negatives") types = { "rbloom": rbloom.Bloom, "pybloomfiltermmap3": pybloomfilter.BloomFilter, "pybloom3": pybloom.BloomFilter, "flor": flor.BloomFilter, "bloomfilter2": bloom_filter2.BloomFilter, } def main(): for name, ty in types.items(): print(f"Running {name}") try: results = timeit.repeat(lambda: run(ty), number=1, repeat=5) extras = "" except Exception as e: results = timeit.repeat(lambda: run_bytes(ty), number=1, repeat=5) extras = f" (via bytes because {e})" avg = sum(results) / len(results) print(f" {avg:6.2f}s{extras}") if __name__ == "__main__": main() KenanHanke-rbloom-b215628/benchmarks/per_operation.py000066400000000000000000000030531505777676000226420ustar00rootroot00000000000000import timeit import time NUMBER = 1000000 def format_time(time_ns: float) -> str: return f"{time_ns / 1000:.04} us" def main(): res = timeit.timeit( setup=f"from rbloom import Bloom; b = Bloom({NUMBER}, 0.01)", stmt="b.add(object())", timer=time.perf_counter_ns, number=NUMBER, ) print("Time to insert an element:") print(format_time(res / NUMBER)) results = timeit.repeat( setup=f"from rbloom import Bloom; b = Bloom({NUMBER}, 0.01); objects = [object() for _ in range({NUMBER})]", stmt="b.update(objects)", timer=time.perf_counter_ns, number=1, repeat=20, ) res = min(results) print("Time to insert each element in a batch:") print(format_time(res / NUMBER)) results = timeit.repeat( setup=f"from rbloom import Bloom; b = Bloom({NUMBER}, 0.01); objects = (object() for _ in range({NUMBER}))", stmt="b.update(objects)", timer=time.perf_counter_ns, number=1, repeat=20, ) res = min(results) print("Time to insert each element in a batch via an iterable:") print(format_time(res / NUMBER)) res = timeit.timeit( setup=f"from rbloom import Bloom; b = Bloom({NUMBER}, 0.01); stored_obj = object(); b.add(stored_obj); b.update(object() for _ in range({NUMBER}))", stmt="stored_obj in b", timer=time.perf_counter_ns, number=NUMBER, ) print("Time to check if an object is present:") print(format_time(res / NUMBER)) if __name__ == "__main__": main() KenanHanke-rbloom-b215628/benchmarks/requirements.in000066400000000000000000000000661505777676000224760ustar00rootroot00000000000000-e ../ pybloomfiltermmap3 pybloom3 Flor bloom-filter2 KenanHanke-rbloom-b215628/benchmarks/requirements.txt000066400000000000000000000005651505777676000227130ustar00rootroot00000000000000# This file was autogenerated by uv via the following command: # uv pip compile ./requirements.in -e ../ # via -r ./requirements.in bitarray==3.7.1 # via pybloom3 bloom-filter2==2.0.0 # via -r ./requirements.in flor==1.1.3 # via -r ./requirements.in pybloom3==0.0.3 # via -r ./requirements.in pybloomfiltermmap3==0.6.0 # via -r ./requirements.in KenanHanke-rbloom-b215628/pyproject.toml000066400000000000000000000013321505777676000202170ustar00rootroot00000000000000[build-system] requires = ["maturin>=1.0,<2.0"] build-backend = "maturin" [project] name = "rbloom" description = "Highly optimized Bloom filter that mimics the Python set API, written in Rust" readme = "README.md" requires-python = ">=3.7" classifiers = [ "Programming Language :: Rust", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", ] dynamic = ["version"] license = "MIT" license-files = ["LICEN[CS]E*"] [project.urls] "Source Code" = "https://github.com/KenanHanke/rbloom" Homepage = "https://github.com/KenanHanke/rbloom" Issues = "https://github.com/KenanHanke/rbloom/issues" [tool.maturin] sdist-include = ["LICENSE", "README.md"] KenanHanke-rbloom-b215628/rbloom.pyi000066400000000000000000000062041505777676000173230ustar00rootroot00000000000000import os from typing import Any, Callable, Iterable, Union, final @final class Bloom: # expected_items: max number of items to be added to the filter # false_positive_rate: max false positive rate of the filter # hash_func: optional argument, see section "Cryptographic security" def __init__(self, expected_items: int, false_positive_rate: float, hash_func=__builtins__.hash) -> None: ... # number of buckets in the filter @property def size_in_bits(self) -> int: ... # retrieve the hash_func given to __init__ @property def hash_func(self) -> Callable[[Any], int]: ... # estimated number of items in the filter @property def approx_items(self) -> float: ... # load from file, see section "Persistence" @classmethod def load(cls, filepath: Union[str, bytes, os.PathLike], hash_func: Callable[[Any], int]) -> Bloom: ... # load from bytes(), see section "Persistence" @classmethod def load_bytes(cls, data: bytes, hash_func: Callable[[Any], int]) -> Bloom: ... # save to file, see section "Persistence" def save(self, filepath: Union[str, bytes, os.PathLike]) -> None: ... # save to a bytes(), see section "Persistence" def save_bytes(self) -> bytes: ... ##################################################################### # ALL SUBSEQUENT METHODS ARE # # EQUIVALENT TO THE CORRESPONDING METHODS # # OF THE BUILT-IN SET TYPE # ##################################################################### def add(self, obj: Any, /) -> None: ... def __contains__(self, obj: Any) -> bool: ... def __bool__(self) -> bool: ... # False if empty def __repr__(self) -> str: ... # basic info def __or__(self, other: Bloom) -> Bloom: ... # self | other def __ior__(self, other: Bloom) -> None: ... # self |= other def __and__(self, other: Bloom) -> Bloom: ... # self & other def __iand__(self, other: Bloom) -> None: ... # self &= other # extension of __or__ def union(self, *others: Union[Iterable, Bloom]) -> Bloom: ... # extension of __ior__ def update(self, *others: Union[Iterable, Bloom]) -> None: ... # extension of __and__ def intersection(self, *others: Union[Iterable, Bloom]) -> Bloom: ... # extension of __iand__ def intersection_update(self, *others: Union[Iterable, Bloom]) -> None: ... # these implement <, >, <=, >=, ==, != def __lt__(self, other: Bloom) -> bool: ... def __gt__(self, other: Bloom) -> bool: ... def __le__(self, other: Bloom) -> bool: ... def __ge__(self, other: Bloom) -> bool: ... def __eq__(self, other: object) -> bool: ... def __ne__(self, other: object) -> bool: ... def issubset(self, other: Bloom, /) -> bool: ... # self <= other def issuperset(self, other: Bloom, /) -> bool: ... # self >= other def clear(self) -> None: ... # remove all items def copy(self) -> Bloom: ... # duplicate self KenanHanke-rbloom-b215628/src/000077500000000000000000000000001505777676000160735ustar00rootroot00000000000000KenanHanke-rbloom-b215628/src/lib.rs000066400000000000000000000530751505777676000172210ustar00rootroot00000000000000use bitline::BitLine; use pyo3::exceptions::{PyTypeError, PyValueError}; use pyo3::prelude::*; use pyo3::sync::PyOnceLock; use pyo3::types::PyType; use pyo3::{basic::CompareOp, types::PyBytes, types::PyTuple, PyTraverseError, PyVisit}; use std::fs::File; use std::io::{Read, Write}; use std::mem; use std::path::PathBuf; #[pyclass(module = "rbloom")] struct Bloom { filter: BitLine, k: u64, // Number of hash functions (implemented via a LCG that uses // the original hash as a seed) hash_func: Option>, } #[pymethods] impl Bloom { #[new] #[pyo3(signature = (expected_items, false_positive_rate, hash_func = None))] fn new( expected_items: u64, false_positive_rate: f64, hash_func: Option>, ) -> PyResult { // Check the inputs if false_positive_rate <= 0.0 || false_positive_rate >= 1.0 { return Err(PyValueError::new_err( "false_positive_rate must be between 0 and 1", )); } if expected_items == 0 { return Err(PyValueError::new_err( "expected_items must be greater than 0", )); } let hash_func = match hash_func { Some(hash_func) if !hash_func.is(builtin_hash_func(hash_func.py())?) => { if !hash_func.is_callable() { return Err(PyTypeError::new_err("hash_func must be callable")); } Some(hash_func.unbind()) } _ => None, }; // Calculate the parameters for the filter let size_in_bits = -(expected_items as f64) * false_positive_rate.ln() / 2.0f64.ln().powi(2); let k = (size_in_bits / expected_items as f64) * 2.0f64.ln(); // Create the filter Ok(Bloom { filter: BitLine::new(size_in_bits as u64)?, k: k as u64, hash_func, }) } /// Number of buckets in the filter #[getter] fn size_in_bits(&self) -> u64 { self.filter.len() } /// Retrieve the hash_func given to __init__ #[getter] fn hash_func<'py>(&self, py: Python<'py>) -> PyResult<&Bound<'py, PyAny>> { match self.hash_func.as_ref() { Some(hash_func) => Ok(hash_func.bind(py)), None => builtin_hash_func(py), } } /// Estimated number of items in the filter #[getter] fn approx_items(&self) -> f64 { let len = self.filter.len() as f64; let bits_set = self.filter.sum() as f64; (len / (self.k as f64) * (1.0 - (bits_set) / len).ln()).abs() } #[pyo3(signature = (o, /))] fn add(&mut self, o: &Bound<'_, PyAny>) -> PyResult<()> { let hash = hash(o, &self.hash_func)?; for index in lcg::generate_indexes(hash, self.k, self.filter.len()) { self.filter.set(index); } Ok(()) } /// Test whether every element in the bloom may be in other /// /// This can have false positives (return true for a bloom which does not /// contain all items in this set), but it will not return a false negative: /// If this returns false, this set contains an element which is not in other #[pyo3(signature = (other, /))] fn issubset(&self, other: &Bound<'_, PyAny>) -> PyResult { self.with_other_as_bloom(other, |other_bloom| { Ok(self.filter.is_subset(&other_bloom.filter)) }) } /// Test whether every element in other may be in self /// /// This can have false positives (return true for a bloom which does not /// contain all items in other), but it will not return a false negative: /// If this returns false, other contains an element which is not in self #[pyo3(signature = (other, /))] fn issuperset(&self, other: &Bound<'_, PyAny>) -> PyResult { self.with_other_as_bloom(other, |other_bloom| { Ok(other_bloom.filter.is_subset(&self.filter)) }) } fn __contains__(&self, o: &Bound<'_, PyAny>) -> PyResult { let hash = hash(o, &self.hash_func)?; for index in lcg::generate_indexes(hash, self.k, self.filter.len()) { if !self.filter.get(index) { return Ok(false); } } Ok(true) } /// Return a new set with elements from the set and all others. #[pyo3(signature = (*others))] fn union<'py>(&self, others: &Bound<'py, PyTuple>) -> PyResult> { let result = self.clone_ref(others.py()); let result = Bound::new(others.py(), result)?; Self::update(&result, others)?; Ok(result) } /// Return a new set with elements common to the set and all others. #[pyo3(signature = (*others))] fn intersection<'py>(&self, others: &Bound<'py, PyTuple>) -> PyResult> { let result = self.clone_ref(others.py()); let result = Bound::new(others.py(), result)?; Self::intersection_update(&result, others)?; Ok(result) } fn __or__(&self, py: Python<'_>, other: &Bloom) -> PyResult { check_compatible(self, other)?; Ok(Bloom { filter: &self.filter | &other.filter, k: self.k, hash_func: self.hash_fn_clone(py), }) } fn __ior__(&mut self, other: &Bloom) -> PyResult<()> { check_compatible(self, other)?; self.filter |= &other.filter; Ok(()) } fn __and__(&self, py: Python<'_>, other: &Bloom) -> PyResult { check_compatible(self, other)?; Ok(Bloom { filter: &self.filter & &other.filter, k: self.k, hash_func: self.hash_fn_clone(py), }) } fn __iand__(&mut self, other: &Bloom) -> PyResult<()> { check_compatible(self, other)?; self.filter &= &other.filter; Ok(()) } // This function has a somewhat unusual signature (taking "self" as a &Bound). This is so it can // check if attempting to call `update` with itself and continue rather than failing because // self cannot be both borrowed mutably for self and in others. #[pyo3(signature = (*others))] fn update(self_bound: &Bound<'_, Self>, others: &Bound<'_, PyTuple>) -> PyResult<()> { let mut self_: PyRefMut<'_, Self> = self_bound.try_borrow_mut()?; for other in others.iter_borrowed() { if other.is(self_bound) { // OR with self is a no-op continue; } // If the other object is a Bloom, use the bitwise union if let Ok(other) = other.downcast::() { let other = other.try_borrow()?; self_.__ior__(&other)?; } // Otherwise, iterate over the other object and add each item else { for obj in other.try_iter()? { self_.add(&obj?)?; } } } Ok(()) } // This function has a somewhat unusual signature (taking "self" as a &Bound). This is so it can // check if attempting to call `update` with itself and continue rather than failing because // self cannot be both borrowed mutably for self and in others. #[pyo3(signature = (*others))] fn intersection_update(self_bound: &Bound<'_, Self>, others: &Bound<'_, PyTuple>) -> PyResult<()> { let mut self_: PyRefMut<'_, Self> = self_bound.try_borrow_mut()?; // Lazily allocated temp bitset let mut temp: Option = None; for other in others.iter_borrowed() { // If the other object is self, AND with self is a no-op if other.is(self_bound) { continue; } // If the other object is a Bloom, use the bitwise intersection if let Ok(other) = other.downcast::() { let other = other.try_borrow()?; self_.__iand__(&other)?; } // Otherwise, iterate over the other object and add each item else { let temp = temp.get_or_insert_with(|| self_.clone_ref(others.py())); temp.clear(); for obj in other.try_iter()? { temp.add(&obj?)?; } self_.__iand__(temp)?; } } Ok(()) } fn clear(&mut self) { self.filter.clear(); } fn copy(&self, py: Python<'_>) -> Bloom { self.clone_ref(py) } fn __repr__(&self) -> String { // Use a format that makes it clear that the object // cannot be reconstructed from the repr format!( "", self.size_in_bits(), self.approx_items() ) } fn __bool__(&self) -> bool { !self.filter.is_empty() } fn __richcmp__(&self, other: &Self, op: CompareOp) -> PyResult { check_compatible(self, other)?; Ok(match op { CompareOp::Eq => self.filter == other.filter, CompareOp::Ne => self.filter != other.filter, CompareOp::Le => self.filter.is_subset(&other.filter), CompareOp::Lt => self.filter.is_strict_subset(&other.filter), CompareOp::Ge => other.filter.is_subset(&self.filter), CompareOp::Gt => other.filter.is_strict_subset(&self.filter), }) } #[classattr] const __hash__: Option> = None; /// Load from a file, see "Persistence" section in the README #[classmethod] fn load( _cls: &Bound<'_, PyType>, filepath: PathBuf, hash_func: &Bound<'_, PyAny>, ) -> PyResult { // check that the hash_func is callable if !hash_func.is_callable() { return Err(PyTypeError::new_err("hash_func must be callable")); } // check that the hash_func isn't the built-in hash function if hash_func.is(builtin_hash_func(hash_func.py())?) { return Err(PyValueError::new_err( "Cannot load a bloom filter that uses the built-in hash function", )); } let hash_func = Some(hash_func.clone().unbind()); let mut file = File::open(filepath)?; let mut k_bytes = [0; mem::size_of::()]; file.read_exact(&mut k_bytes)?; let k = u64::from_le_bytes(k_bytes); let filter = BitLine::load(&mut file)?; Ok(Bloom { filter, k, hash_func, }) } /// Load from a bytes(), see "Persistence" section in the README #[classmethod] fn load_bytes( _cls: &Bound<'_, PyType>, bytes: &[u8], hash_func: &Bound<'_, PyAny>, ) -> PyResult { // check that the hash_func is callable if !hash_func.is_callable() { return Err(PyTypeError::new_err("hash_func must be callable")); } // check that the hash_func isn't the built-in hash function if hash_func.is(builtin_hash_func(hash_func.py())?) { return Err(PyValueError::new_err( "Cannot load a bloom filter that uses the built-in hash function", )); } let hash_func = Some(hash_func.clone().unbind()); let k_bytes: [u8; mem::size_of::()] = bytes[0..mem::size_of::()] .try_into() .expect("slice with incorrect length"); let k = u64::from_le_bytes(k_bytes); let filter = BitLine::load_bytes(&bytes[mem::size_of::()..])?; Ok(Bloom { filter, k, hash_func, }) } /// Save to a file, see "Persistence" section in the README fn save(&self, filepath: PathBuf) -> PyResult<()> { if self.hash_func.is_none() { return Err(PyValueError::new_err( "Cannot save a bloom filter that uses the built-in hash function", )); } let mut file = File::create(filepath)?; file.write_all(&self.k.to_le_bytes())?; self.filter.save(&mut file)?; Ok(()) } /// Save to a byte(), see "Persistence" section in the README fn save_bytes<'py>(&self, py: Python<'py>) -> PyResult> { const K_SIZE: usize = mem::size_of::(); if self.hash_func.is_none() { return Err(PyValueError::new_err( "Cannot save a bloom filter that uses the built-in hash function", )); } debug_assert_eq!(K_SIZE, self.k.to_le_bytes().len()); let len = K_SIZE + self.filter.bits().len(); PyBytes::new_with(py, len, |data| { data[..K_SIZE].copy_from_slice(&self.k.to_le_bytes()); data[K_SIZE..].copy_from_slice(self.filter.bits()); Ok(()) }) } fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { visit.call(&self.hash_func)?; Ok(()) } } // Non-python methods impl Bloom { fn hash_fn_clone(&self, py: Python<'_>) -> Option> { self.hash_func.as_ref().map(|f| f.clone_ref(py)) } fn clone_ref(&self, py: Python<'_>) -> Self { Bloom { filter: self.filter.clone(), k: self.k, hash_func: self.hash_fn_clone(py), } } fn zeroed_clone(&self, py: Python<'_>) -> Bloom { Bloom { filter: BitLine::new(self.filter.len()).unwrap(), k: self.k, hash_func: self.hash_fn_clone(py), } } /// Extract other as a bloom, or iterate other, and add all items to a temporary bloom fn with_other_as_bloom( &self, other: &Bound<'_, PyAny>, f: impl FnOnce(&Bloom) -> PyResult, ) -> PyResult { match other.downcast::() { Ok(o) => { let o = o.try_borrow()?; check_compatible(self, &o)?; f(&o) } Err(_) => { let mut other_bloom = self.zeroed_clone(other.py()); for obj in other.try_iter()? { other_bloom.add(&obj?)?; } f(&other_bloom) } } } } /// This is a primitive BitVec-like structure that uses a `Box<[u8]>` as /// the backing store; it exists here to avoid the need for a dependency /// on bitvec and to act as a container around all the bit manipulation. /// Indexing is done using u64 to avoid address space issues on 32-bit /// systems, which would otherwise limit the size to 2^32 bits (512MB). /// Using u8 for the backing store simplifies file I/O as well as file /// portability across systems, and the performance is equivalent to /// using usize, even though the latter is arguably more elegant. mod bitline { use pyo3::exceptions::PyValueError; use pyo3::prelude::*; use std::fs::File; use std::io::{Read, Write}; #[inline(always)] fn bit_idx(idx: u64) -> Option<(usize, u32)> { let (q, r) = (idx / 8, idx % 8); Some((q.try_into().ok()?, r.try_into().ok()?)) } #[derive(Clone, PartialEq, Eq)] pub struct BitLine { bits: Box<[u8]>, } impl BitLine { pub fn new(size_in_bits: u64) -> PyResult { match bit_idx(size_in_bits) { Some((q, r)) => { let size = if r == 0 { q } else { q + 1 }; Ok(Self { bits: vec![0; size].into_boxed_slice(), }) } None => Err(PyValueError::new_err("too many bits")), } } /// Make sure that index is less than len when calling this! pub fn set(&mut self, index: u64) { let (idx, offset) = bit_idx(index).unwrap(); self.bits[idx] |= 1 << offset; } /// Make sure that index is less than len when calling this! pub fn get(&self, index: u64) -> bool { let (idx, offset) = bit_idx(index).unwrap(); self.bits[idx] & (1 << offset) != 0 } /// Returns the number of bits in the BitLine pub fn len(&self) -> u64 { self.bits.len() as u64 * 8 } pub fn clear(&mut self) { self.bits.fill(0); } pub fn sum(&self) -> u64 { self.bits.iter().map(|x| x.count_ones() as u64).sum() } pub fn is_empty(&self) -> bool { self.bits.iter().all(|&word| word == 0) } pub fn is_subset(&self, other: &BitLine) -> bool { all_pairs(self, other, |lhs, rhs| (lhs | rhs) == rhs) } pub fn is_strict_subset(&self, other: &BitLine) -> bool { let mut is_equal = true; let is_subset = all_pairs(self, other, |lhs, rhs| { is_equal &= lhs == rhs; (lhs | rhs) == rhs }); is_subset && !is_equal } /// Reads the given file from the current position to the end and /// returns a BitLine containing the data. pub fn load(file: &mut File) -> PyResult { let mut bits = Vec::new(); file.read_to_end(&mut bits)?; Ok(Self { bits: bits.into_boxed_slice(), }) } /// Given the provided [u8] /// returns a BitLine containing the data. pub fn load_bytes(bytes: &[u8]) -> PyResult { let bits = bytes.to_vec(); Ok(Self { bits: bits.into_boxed_slice(), }) } /// Writes the BitLine to the given file from the current position. pub fn save(&self, file: &mut File) -> PyResult<()> { file.write_all(&self.bits)?; Ok(()) } pub fn bits(&self) -> &[u8] { &self.bits } } fn all_pairs(lhs: &BitLine, rhs: &BitLine, mut f: impl FnMut(u8, u8) -> bool) -> bool { lhs.bits .iter() .zip(rhs.bits.iter()) .all(move |(&lhs, &rhs)| f(lhs, rhs)) } impl std::ops::BitAnd for BitLine { type Output = Self; fn bitand(mut self, rhs: Self) -> Self::Output { self &= rhs; self } } impl std::ops::BitAnd for &BitLine { type Output = BitLine; fn bitand(self, rhs: Self) -> Self::Output { let mut result = self.clone(); result &= rhs; result } } impl std::ops::BitAndAssign for BitLine { fn bitand_assign(&mut self, rhs: Self) { *self &= &rhs; } } impl std::ops::BitAndAssign<&BitLine> for BitLine { fn bitand_assign(&mut self, rhs: &Self) { for (lhs, rhs) in self.bits.iter_mut().zip(rhs.bits.iter()) { *lhs &= rhs; } } } impl std::ops::BitOr for BitLine { type Output = Self; fn bitor(mut self, rhs: Self) -> Self::Output { self |= rhs; self } } impl std::ops::BitOr for &BitLine { type Output = BitLine; fn bitor(self, rhs: Self) -> Self::Output { let mut result = self.clone(); result |= rhs; result } } impl std::ops::BitOrAssign for BitLine { fn bitor_assign(&mut self, rhs: Self) { *self |= &rhs; } } impl std::ops::BitOrAssign<&BitLine> for BitLine { fn bitor_assign(&mut self, rhs: &Self) { for (lhs, rhs) in self.bits.iter_mut().zip(rhs.bits.iter()) { *lhs |= rhs; } } } } /// This implements a linear congruential generator that is /// used to distribute entropy from the hash over multiple ints. mod lcg { pub struct Random { state: u128, } impl Iterator for Random { type Item = u64; fn next(&mut self) -> Option { self.state = self .state .wrapping_mul(47026247687942121848144207491837418733) .wrapping_add(1); Some((self.state >> 32) as Self::Item) } } pub fn distribute_entropy(hash: i128) -> Random { Random { state: hash as u128, } } pub fn generate_indexes(hash: i128, k: u64, len: u64) -> impl Iterator { distribute_entropy(hash) .take(k as usize) .map(move |x: u64| x % len) } } fn hash(o: &Bound<'_, PyAny>, hash_func: &Option>) -> PyResult { match hash_func { Some(hash_func) => { let hash_func = hash_func.bind(o.py()); let hash = hash_func.call1((o,))?; Ok(hash.extract()?) } None => Ok(o.hash()? as i128), } } fn check_compatible(a: &Bloom, b: &Bloom) -> PyResult<()> { if a.k != b.k || a.filter.len() != b.filter.len() { return Err(PyValueError::new_err( "size and max false positive rate must be the same for both filters", )); } // now only the hash function can be different match (&a.hash_func, &b.hash_func) { (Some(lhs), Some(rhs)) if lhs.is(rhs) => {} (&None, &None) => {} _ => { return Err(PyValueError::new_err( "Bloom filters must have the same hash function", )) } } Ok(()) } fn builtin_hash_func(py: Python<'_>) -> PyResult<&Bound<'_, PyAny>> { static HASH_FUNC: PyOnceLock> = PyOnceLock::new(); let res = HASH_FUNC.get_or_try_init(py, || -> PyResult<_> { let builtins = PyModule::import(py, "builtins")?; Ok(builtins.getattr("hash")?.unbind()) })?; Ok(res.bind(py)) } #[pymodule] fn rbloom(m: &Bound<'_, PyModule>) -> PyResult<()> { m.add_class::()?; Ok(()) } KenanHanke-rbloom-b215628/tests/000077500000000000000000000000001505777676000164465ustar00rootroot00000000000000KenanHanke-rbloom-b215628/tests/api_test.py000077500000000000000000000101311505777676000206270ustar00rootroot00000000000000#!/usr/bin/env python3 import gc import weakref from rbloom import Bloom from hashlib import sha256 from pickle import dumps import os def test_bloom(bloom: Bloom): assert not bloom assert bloom.approx_items == 0.0 bloom.add('foo') assert bloom assert bloom.approx_items > 0.0 bloom.add('bar') assert 'foo' in bloom assert 'bar' in bloom assert 'baz' not in bloom bloom.update(['baz', 'qux']) assert 'baz' in bloom assert 'qux' in bloom other = bloom.copy() assert other == bloom assert other is not bloom other.clear() assert not other assert other.approx_items == 0.0 other.update(['foo', 'bar', 'baz', 'qux']) assert other == bloom other.update(str(i).encode()*500 for i in range(100000)) for i in range(100000): assert str(i).encode()*500 in other assert bloom != other assert bloom & other == bloom assert bloom | other == other bloom &= other assert bloom < other orig = bloom.copy() bloom |= other assert bloom == other assert bloom > orig assert bloom >= orig assert bloom.issuperset(other) assert orig <= bloom assert orig.issubset(bloom) assert bloom >= bloom assert bloom.issuperset(bloom) assert bloom <= bloom assert bloom.issubset(bloom) bloom = orig.copy() bloom.update(other) assert bloom == other assert bloom > orig bloom = orig.copy() assert other == bloom.union(other) assert bloom == bloom.intersection(other) bloom.intersection_update(other) assert bloom == orig # TEST PERSISTENCE if not bloom.hash_func is hash: # find a filename that doesn't exist i = 0 while os.path.exists(f'UNIT_TEST_{i}.bloom'): i += 1 filename = f'test{i}.bloom' try: # save and load bloom.save(filename) bloom2 = Bloom.load(filename, bloom.hash_func) assert bloom == bloom2 finally: # remove the file os.remove(filename) # TEST bytes PERSISTENCE bloom_bytes = bloom.save_bytes() assert type(bloom_bytes) == bytes bloom3 = Bloom.load_bytes(bloom_bytes, bloom.hash_func) assert bloom == bloom3 def sha_based(obj): h = sha256(dumps(obj)).digest() return int.from_bytes(h[:16], "big") - 2**127 def circular_ref(): def loop_hash_func(x): return sha_based(x) weak_ref = weakref.ref(loop_hash_func) bloom = Bloom(1000, 0.1, hash_func=loop_hash_func) assert gc.get_referents(bloom) == [loop_hash_func] loop_hash_func.bloom = bloom del bloom del loop_hash_func gc.collect() assert weak_ref() is None def operations_with_self(): bloom = Bloom(1000, 0.1) bloom.add('foo') assert 'foo' in bloom bloom |= bloom bloom &= bloom bloom.update(bloom) bloom.update(bloom, bloom) bloom.update(bloom, ['bob'], bloom) assert 'foo' in bloom assert 'bob' in bloom bloom.intersection_update(bloom) bloom.intersection_update(bloom, bloom) bloom.intersection_update(bloom, ['foo'], bloom) assert 'foo' in bloom assert 'bob' not in bloom assert bloom == bloom assert bloom <= bloom assert bloom >= bloom assert not (bloom > bloom) assert not (bloom < bloom) assert bloom.issubset(bloom) assert bloom.issuperset(bloom) assert bloom.union(bloom) == bloom assert bloom.union(bloom, bloom) == bloom assert bloom.intersection(bloom) == bloom assert bloom.intersection(bloom, bloom) == bloom def api_suite(): assert repr(Bloom(27_000, 0.0317)) == "" assert Bloom(1140, 0.999).hash_func == hash assert Bloom(102, 0.01, hash_func=hash).hash_func is hash assert Bloom(103100, 0.51, hash_func=sha_based).hash_func is sha_based test_bloom(Bloom(13242, 0.0000001)) test_bloom(Bloom(9874124, 0.01, hash_func=sha_based)) test_bloom(Bloom(2837, 0.5, hash_func=hash)) circular_ref() operations_with_self() print('All API tests passed') if __name__ == '__main__': api_suite()