pax_global_header00006660000000000000000000000064152406740530014520gustar00rootroot0000000000000052 comment=1619eeeab589af55d7b59e4e3e2be8719d017940 pcapy-ng-2.1.0/000077500000000000000000000000001524067405300132365ustar00rootroot00000000000000pcapy-ng-2.1.0/.github/000077500000000000000000000000001524067405300145765ustar00rootroot00000000000000pcapy-ng-2.1.0/.github/workflows/000077500000000000000000000000001524067405300166335ustar00rootroot00000000000000pcapy-ng-2.1.0/.github/workflows/tests.yml000066400000000000000000000033421524067405300205220ustar00rootroot00000000000000name: Tests # Build the extension from source and run the test suite on every supported CPython. # Offline captures only (tests read .pcap files), so no privileges are needed. on: push: branches: [master] pull_request: workflow_dispatch: jobs: test: name: ${{ matrix.os }} / Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [ubuntu-latest, macos-latest] python-version: ["3.9", "3.10", "3.11", "3.12", "3.13", "3.14"] steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install libpcap headers (Linux) if: runner.os == 'Linux' run: sudo apt-get update && sudo apt-get install -y libpcap-dev - name: Build and install run: | python -m pip install --upgrade pip python -m pip install . - name: Run tests run: | python tests/wheel_smoketest.py tests/96pings.pcap python -m unittest discover -s tests -p 'test_fast.py' -v # pcapytests.py::testEOFValue asserts that sys.getrefcount(None) is unchanged across the # test body. That is only stable where None is immortal (CPython 3.12+); on older # interpreters the count drifts by a few references from unittest's own machinery, so the # assertion fails non-deterministically and says nothing about pcapy. - name: Run classic API tests if: matrix.python-version != '3.9' && matrix.python-version != '3.10' && matrix.python-version != '3.11' working-directory: tests # pcapytests.py opens 96pings.pcap relative to the cwd run: python pcapytests.py -v pcapy-ng-2.1.0/.github/workflows/wheels.yml000066400000000000000000000057101524067405300206500ustar00rootroot00000000000000name: Build wheels # Produce self-contained, precompiled wheels (libpcap vendored in) plus an sdist, so # downstream users (e.g. Maltrail sensors) can `pip install pcapy-ng` with no C toolchain # or libpcap-dev at deploy time. # # Triggers: # * push of a version tag ("2.0.1" or "v2.0.1") -> build + test wheels and sdist (artifacts only) # * manual run -> same; set "publish" to upload to PyPI # * publishing a GitHub Release -> build + test + upload to PyPI # # Uploading requires PyPI Trusted Publishing to be configured for this repository plus a # GitHub environment named "pypi"; until then the publish job simply never runs. on: push: tags: - "v[0-9]+.[0-9]+.[0-9]+" - "[0-9]+.[0-9]+.[0-9]+" release: types: [published] workflow_dispatch: inputs: publish: description: "Upload the built artifacts to PyPI" type: boolean default: false jobs: build_wheels: name: Wheels on ${{ matrix.os }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: - ubuntu-latest # linux x86_64 (manylinux + musllinux) - ubuntu-24.04-arm # linux aarch64 - macos-15-intel # macOS x86_64 (macos-13 was retired in December 2025) - macos-latest # macOS arm64 steps: - uses: actions/checkout@v4 - name: Build wheels uses: pypa/cibuildwheel@v4.2.0 # all configuration lives in [tool.cibuildwheel] in pyproject.toml - uses: actions/upload-artifact@v4 with: name: wheels-${{ matrix.os }} path: ./wheelhouse/*.whl build_sdist: name: Build sdist runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: "3.x" - name: Install build deps run: | sudo apt-get update && sudo apt-get install -y libpcap-dev python -m pip install --upgrade build twine - name: Build sdist run: python -m build --sdist - name: Check metadata run: python -m twine check dist/*.tar.gz - name: Install from sdist and run the test suite run: | python -m pip install dist/*.tar.gz python tests/wheel_smoketest.py tests/96pings.pcap python -m unittest discover -s tests -p test_fast.py - uses: actions/upload-artifact@v4 with: name: sdist path: ./dist/*.tar.gz publish: name: Publish to PyPI needs: [build_wheels, build_sdist] if: github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.publish) runs-on: ubuntu-latest environment: pypi permissions: id-token: write # PyPI Trusted Publishing steps: - uses: actions/download-artifact@v4 with: path: dist merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 pcapy-ng-2.1.0/.gitignore000077500000000000000000000000731524067405300152310ustar00rootroot00000000000000*.py[co] __pycache__/ *.so *.pyd *.egg-info /build/ /dist/ pcapy-ng-2.1.0/ChangeLog000077500000000000000000000177711524067405300150300ustar00rootroot000000000000002026-08-17 Miroslav Stampar * Version 2.1.0 tagged (one behaviour change, one dropped support claim): * loop_filtered(), loop_to_buffer() and set_fanout() accept keyword arguments. Their docstrings and the README always documented calls like loop_filtered(..., flow_cutoff=3), but the methods were METH_VARARGS, so every such call raised TypeError. * Rewrote the flow-table key. The old fold multiplied the destination address by 40503, which kept it below 2^48, so the top bits of the 64-bit key carried no destination entropy; two flows could then share one counter and a real FLOW_HEAD would be dropped - the one failure mode the design excludes (slot collisions are benign, they only restart a counter). The key is now a splitmix64 finalizer over the whole 5-tuple, so every input bit reaches all 64 output bits. Flow-head counts move by a fraction of a percent. * Fixed two shifts into the sign bit of a signed int (undefined behaviour): the source port in the old flow key, and PACKET_FANOUT_FLAG_DEFRAG (0x8000) in set_fanout(). * python_requires is now ">=3.9", matching the CI matrix, the wheels and the classifiers; it previously admitted 2.7 and 3.8, which nothing builds or tests. The Python 2 code paths in the C sources are untouched, just no longer advertised. * benchmarks/results/ (results.csv, results.json, SUMMARY.md) is committed instead of git-ignored, so every published ratio can be checked against the run behind it. 2026-08-15 Miroslav Stampar * Version 2.0.2 tagged (bugfix release; no API or behaviour changes): * Fixed the slot index of the loop_filtered()/loop_to_buffer() IPv4 address set. It multiplied the address and then kept the LOW bits of the product, which discarded the mixing, so consecutive addresses (an expanded CIDR block) collapsed onto one slot and linear probing degenerated into a scan: ~6850 probes per lookup on a 100k-entry sequential set, against ~1.3 for random addresses. The index now takes the HIGH bits of the product, which costs the same and brings sequential sets to ~1.0 probes; measured 2.6x-24x faster on sequential sets of 1k-1M entries, with random sets unchanged. Matching results, the packed addr_set format and the API are unchanged. * Added address-set membership tests covering large sequential ranges (up to 1M entries), random sets, expanded CIDR blocks, duplicates and the 0.0.0.0 sentinel. 2026-08-15 Miroslav Stampar * Version 2.0.1 tagged (packaging, CI and documentation release; no API changes): * PyPI metadata modernized: long description is now README.md (rendered as Markdown), plus an accurate summary, keywords, project URLs, SPDX license, classifiers and a python_requires of 2.7 and 3.8+. * Packaging fix: data_files no longer globs directories, so a leftover tests/__pycache__ can no longer abort the build with "can't copy 'tests/__pycache__'". * Python 3.14 supported: added to the cibuildwheel matrix (CPython 3.9-3.14) and to the test matrix; builds and passes the full test suite. * Release workflow fixed: it now triggers on this project's bare version tags, builds and tests wheels for Linux (x86_64/aarch64, glibc and musl) and macOS (x86_64/arm64), verifies the sdist, and uploads to PyPI only from a published GitHub Release or an explicit manual dispatch. New tests workflow builds and tests every commit. * Documentation rewritten to describe the full 2.0 API surface - loop_filtered(), filtered_stats(), loop_to_buffer(), next_batch(), set_fanout(), flow-head tracking, large IPv4 address-set matching and the built-in security profile - and to state what is actually shipped (no Windows wheels; Windows builds from source with the Npcap SDK). * Dropped the obsolete AppVeyor configuration (Python 2.7/3.4/3.5 against WinPcap). * Fix: pcap_dealloc() dropped the new reference to None returned by p_close(), leaking one reference to None per destroyed Reader (invisible since None became immortal in CPython 3.12). 2026-06-26 Miroslav Stampar * Version 2.0.0 tagged with major new capabilities (in-C classification + scaling): * loop_filtered(): classify each packet in C and invoke the callback only for admitted classes; provably-inert noise never builds a Python object. Generic profile (0=OTHER, 1=FLOW_HEAD, 2=SET_MATCH) and a built-in security profile (profile=1). * loop_to_buffer(): same classification, GIL released for the whole loop, writes admitted packets into a shared buffer for parallel worker draining (no per-packet Python). * filtered_stats(): live per-class counter snapshot, pollable from another thread. * next_batch(): capture up to N packets in one call (amortized C<->Python boundary). * set_fanout(group_id, fanout_type=PACKET_FANOUT_HASH) + PACKET_FANOUT_* constants: join a live capture to a Linux kernel PACKET_FANOUT group to scale capture across multiple sockets/threads on one interface (flow-hashed, no duplicate capture). * Classic Reader API (loop/next/dispatch/setfilter/...) unchanged and backward compatible. 2018-06-17 Alberto Solino * Version 0.11.4 tagged with the following changes * bpfobject support get_bpf method (by @palaviv) * Monitor mode support & missing methods fix (by @gpotter2) * Documentation update (@gpotter2) 2018-03-26 Alberto Solino * Version 0.11.2 tagged with the following substantial changes (since version 0.11.11): * Exporting pcap_get_selectable_fd as getfd (by @geekhckr) * Add a close method & context manager to Reader (by @segevfiner) 2017-01-16 Martin Balao * Version 0.11.1 tagged with the following changes (since version 0.10.10): * AppVeyor integration (continuous integration service) * @takluyver contribution * Close method for Pcap Dumper * @JinBlack contribution * Pcap stats added * @duggan contribution 2015-05-19 Andres Blanco * Added sendpacket support by default 2010-08-25 Ezequiel Gutesman * removed check for pcap_sendpacket function 2010-07-30 Ezequiel Gutesman * pcapy.cc: fixed bug in initpcapy, Pkthdr_type.ob_type and Pdumpertype.ob_type were not set. See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=589956 (reported by Piotr Lewandowski) 2007-03-27 Max Caceres * pcapobj.cc: wrapped pcap_sendpacket for sending packets in all platforms. 2005-12-09 Max Caceres * pcapy.cc: open_live doesn't raise an exception anymore when network/netmask information is not available (i.e. when sniffing on devices without an assigned IP address). Reported by Scott Raynel. 2005-09-14 Max Caceres * pcapobj.cc: fixed bug where exceptions thrown inside a callback where not handled properly. Fix relies on fairly new pcap_breakloop function. * pcapy.cc: in Windows uses findalldevs() packaged with new Winpcap 3.1 2005-08-15 Max Caceres * pcapy.cc: fixed bug where getnet() and getmask() always returned 0.0.0.0 (reported by Jeff Connelly) 2005-07-28 Max Caceres * pcapobj.cc: fixed a memory leak in PythonCallback (reported by fraca7 _at_ free.fr). 2004-01-05 Javier Kohen * setup.py: force linking with G++ when using GCC, to avoid a missing symbol in some setups. 2003-11-28 Javier Kohen * dcerpc_v4.py: Fixed self.bind variable having the same name than a method. pcapy-ng-2.1.0/LICENSE000077500000000000000000000041461524067405300142530ustar00rootroot00000000000000Licencing --------- The Apache Software License, Version 1.1 Copyright (c) 2000 The Apache Software Foundation. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. 3. The end-user documentation included with the redistribution, if any, must include the following acknowledgment: "This product includes software developed by the Apache Software Foundation (http://www.apache.org/)." Alternately, this acknowledgment may appear in the software itself, if and wherever such third-party acknowledgments normally appear. 4. The names "Apache" and "Apache Software Foundation" must not be used to endorse or promote products derived from this software without prior written permission. For written permission, please contact apache@apache.org. 5. Products derived from this software may not be called "Apache", nor may "Apache" appear in their name, without prior written permission of the Apache Software Foundation. THIS SOFTWARE IS PROVIDED ``AS IS'' AND ANY EXPRESSED OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE APACHE SOFTWARE FOUNDATION OR ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. pcapy-ng-2.1.0/MANIFEST.in000077500000000000000000000004101524067405300147720ustar00rootroot00000000000000include MANIFEST.in include LICENSE include ChangeLog include README include README.md include pcapy.xml include pcapy.html include *.h include *.cc include pyproject.toml recursive-include win32 *.cc recursive-include tests * recursive-include examples *.py *.md pcapy-ng-2.1.0/README000077500000000000000000000043241524067405300141240ustar00rootroot00000000000000This file is a plain-text extract of the documentation at https://github.com/stamparm/pcapy-ng/blob/master/README.md (the full documentation, with examples, is in README.md). What is Pcapy-NG? ================= Pcapy-NG is a Python extension module that lets Python programs use the libpcap packet-capture library. It is a maintained replacement for Pcapy, which is no longer maintained and stopped working on Python 3.10. The classic Pcapy API (open_live, open_offline, loop, next, setfilter, dump_open, ...) is unchanged, so existing Pcapy code keeps working. On top of that it provides an optional set of capture primitives for programs where per-packet Python work is the bottleneck: * loop_filtered() - classify packets in C, invoke the callback only for the classes you admit (flow heads, large IPv4 address-set matches, or a built-in security profile). * loop_to_buffer() - same classification, but writes admitted packets into a buffer you provide, with the GIL released. * filtered_stats() - live per-class counters, pollable from another thread. * next_batch() - read up to N packets in a single call. * set_fanout() - join a live handle to a Linux PACKET_FANOUT group so several handles can share one interface's traffic. Setup ===== Install from PyPI: pip install pcapy-ng Releases from 2.0.1 on provide Linux and macOS wheels for CPython 3.9-3.14 with libpcap vendored in. There are no Windows wheels; on Windows, and on any other platform or interpreter, pip builds from the sdist. Building from source requires: * A Python interpreter. CPython 3.9 and newer are supported. * A C++ compiler. * Libpcap and its headers (libpcap-dev / libpcap-devel). On Windows, the Npcap SDK, with WPDPACK_BASE pointing at it. To build and install a checkout: pip install . Licensing ========= This software is provided under the Apache Software License. See the accompanying LICENSE file for more information. Contact Us ========== Whether you want to report a bug, send a patch or give some suggestions on this package, drop a few lines at contact@sekuripy.hr or open an issue at https://github.com/stamparm/pcapy-ng/issues. pcapy-ng-2.1.0/README.md000077500000000000000000000435711524067405300145320ustar00rootroot00000000000000## What is Pcapy-NG? ## [![Tests](https://github.com/stamparm/pcapy-ng/actions/workflows/tests.yml/badge.svg)](https://github.com/stamparm/pcapy-ng/actions/workflows/tests.yml) Pcapy-NG is a Python extension module that lets Python programs use the [libpcap](https://www.tcpdump.org/) packet-capture library. It is a maintained replacement for [Pcapy](https://github.com/helpsystems/pcapy), which is no longer maintained and stopped working on Python 3.10 ([issue](https://github.com/helpsystems/pcapy/issues/70)). The classic Pcapy API — `open_live`, `open_offline`, `loop`, `next`, `setfilter`, `dump_open` and the rest — is unchanged, so existing Pcapy code keeps working; it builds on Linux, macOS and Windows (libpcap / Npcap), on CPython 3.9 and newer. On top of that it provides an optional set of capture primitives for programs where per-packet Python work is the bottleneck: `loop_filtered` classifies packets in C and invokes the callback only for the classes you admit, `loop_to_buffer` writes admitted packets into a buffer you own with the GIL released, `next_batch` returns many packets per call, `filtered_stats` exposes live counters, and `set_fanout` joins a live handle to a Linux `PACKET_FANOUT` group so several handles can share one interface's traffic. The classifier can track flow heads (the first N packets of each flow), match against arbitrarily large IPv4 address sets, and run a built-in security profile that tags DNS, HTTP, TLS/QUIC handshakes and TCP SYNs. All of it is opt-in — ignore it and pcapy-ng behaves exactly like Pcapy. --- ## Contents ## - [Install](#install) - [Quick start](#quick-start) - [`loop_filtered`: filtering in C](#loop_filtered-filtering-in-c) - [Why not just a BPF filter?](#why-not-just-a-bpf-filter) - [Classes and the admit mask](#classes-and-the-admit-mask) - [Flow heads: the start of every connection](#flow-heads-the-start-of-every-connection) - [Address-set matching](#address-set-matching) - [`loop_to_buffer`: filling a shared buffer](#loop_to_buffer-filling-a-shared-buffer) - [Live counters and non-Ethernet links](#live-counters-and-non-ethernet-links) - [Built-in security profile](#built-in-security-profile) - [`next_batch`: many packets per call](#next_batch-many-packets-per-call) - [`set_fanout`: several handles on one interface](#set_fanout-several-handles-on-one-interface) - [API reference](#api-reference) - [Examples](#examples) - [Building & compatibility](#building--compatibility) - [License & credits](#license--credits) --- ## Install ## ```sh pip install pcapy-ng ``` Releases from 2.0.1 on ship an sdist plus Linux (x86_64/aarch64, glibc and musl) and macOS (x86_64/arm64) wheels for CPython 3.9–3.14; those wheels vendor libpcap, so nothing is compiled at install time. Everywhere else — Windows, BSD, and on 2.0.0 and older — pip falls back to the sdist, which needs a C++ compiler and libpcap headers (`libpcap-dev` / `libpcap-devel`; on Windows the [Npcap SDK](https://npcap.com/#download), with `WPDPACK_BASE` pointing at it). To build a checkout: ```sh pip install . ``` Python 2.7 is not a supported target: `python_requires` is `>=3.9`, nothing in CI tests it and no wheel is built for it. The C sources do still carry their Python 2 code paths, so a 2.7 build can be driven the old way — `python setup.py install`, since the PEP 517 build needs `setuptools >= 61`, which is Python 3 only — but that is unsupported and untested. --- ## Quick start ## ```python import pcapy print(pcapy.findalldevs()) # list devices cap = pcapy.open_live("eth0", 65535, True, 100) # device, snaplen, promiscuous, timeout(ms) cap.setfilter("tcp port 80") # optional BPF filter (kernel-side) while True: hdr, data = cap.next() if hdr is None: break print(hdr.getts(), hdr.getlen(), len(data)) cap.loop(-1, lambda hdr, data: print(len(data))) # ...or use a callback cap = pcapy.open_offline("capture.pcap") # read a saved capture dumper = cap.dump_open("out.pcap") # write packets out ``` `hdr` is a `Pkthdr` with `getts()` → `(seconds, microseconds)`, `getlen()` (wire length) and `getcaplen()` (captured length). This is the original Pcapy surface, unchanged. --- ## `loop_filtered`: filtering in C ## ```python cap.loop_filtered(cnt, callback, admit_mask=7, addr_set=b"", flow_cutoff=0) ``` Everything after `callback` is optional and can be passed positionally or by keyword. `loop_filtered` classifies each packet in C, hands your `callback(hdr, data, cls)` only the packets you asked for, and drops the rest **without building a Python object for them**. On a busy link where most traffic is bulk you don't care about, that per-packet object + callback cost is the thing that makes a pure-Python sniffer fall behind; this avoids paying it. ### Why not just a BPF filter? ### Fair question — and for a lot of cases the answer is "use BPF." `setfilter("udp port 53")` runs in the **kernel** and is the right tool for stateless port/protocol matching. You can (and should) keep using it; `loop_filtered` runs *after* it. `loop_filtered` exists for the two things a BPF program **can't** do, because BPF is stateless and size-limited: 1. **Flow state** — "give me the first N packets of each connection, drop the rest." BPF can't track flows. This is how you grab the *start* of every conversation (where the useful bytes are) while ignoring the bulk that follows. 2. **Large set membership** — "deliver packets whose source or destination is in this set of addresses," where the set has thousands or millions of entries. That won't fit in a BPF program; here it's an average O(1) hash lookup. If you only need stateless port/proto matching, use BPF. Reach for `loop_filtered` when you need flow state or set membership (and you can combine both with a BPF prefilter). ### Classes and the admit mask ### Each delivered packet comes with a small integer `cls` saying why it was kept: | `cls` | Name | Meaning | |------|------|---------| | `0` | `OTHER` | didn't match `flow_cutoff` or `addr_set` | | `1` | `FLOW_HEAD` | within the first `flow_cutoff` packets of its flow | | `2` | `SET_MATCH` | source or destination is in `addr_set` | Classes are **mutually exclusive** — each packet gets exactly one `cls` and increments exactly one counter. When a packet could be more than one, precedence is **`SET_MATCH` > `FLOW_HEAD` > `OTHER`** (a flow's first packet that is also a set hit is reported as `SET_MATCH`). `admit_mask` is a bitmask **over the class number** — bit `cls` admits that class; the rest are dropped in C. Default `7` (= `1<<0 | 1<<1 | 1<<2`) admits all three, so a bare `loop_filtered` behaves like `loop` but tagged + counted. To keep only flow heads and set matches, drop `OTHER`: ```python OTHER, FLOW_HEAD, SET_MATCH = 0, 1, 2 admit = (1 << FLOW_HEAD) | (1 << SET_MATCH) stats = cap.loop_filtered(-1, my_cb, admit, addr_set, flow_cutoff=3) # stats -> (admitted, dropped, n_other, n_flow_head, n_set_match) ``` The return value is `(admitted, dropped, )`. The per-class counts are independent of `admit_mask`, so `admit_mask=0` profiles your traffic with no Python work at all. ### Flow heads: the start of every connection ### `flow_cutoff=N` delivers the first `N` packets of each flow (TCP **and** UDP) as `FLOW_HEAD` and lets you ignore the rest. The opening packets of a connection are where the interesting metadata lives — the requested hostname inside a TLS handshake, the first request line, the greeting of a text protocol — so this captures that and sheds the bulk payload after it. ```python def on_head(hdr, data, cls): # data is the start of a connection; parse out whatever you need (e.g. the TLS server name) print("connection start:", len(data), "bytes") cap.loop_filtered(-1, on_head, (1 << 1), flow_cutoff=3) # FLOW_HEAD only ``` **What "flow" means here.** A flow is keyed by the **directional** 5-tuple `(src, dst, src_port, dst_port, protocol)` — so the two directions of a connection are two flows, and `flow_cutoff=N` yields up to `N` packets *per direction*. There is no TCP-state tracking (it counts packets per key) and no time-based expiry; the flow table is a fixed ~12 MB hash (about 1M entries, allocated only when `flow_cutoff > 0`) that evicts the least-recently-seen key on collision — so memory is bounded even under high-cardinality UDP, at the cost of an occasional re-count when a long-idle flow is evicted. Eviction can only make a flow look *new* again (an extra head), never hide one: the slot stores the full 64-bit key of the 5-tuple, and that key is an avalanching hash, so two different flows sharing one counter — the only way a real flow head could be missed — is a ~2⁻⁶⁴ event rather than a property of the fold. IP fragments are not reassembled (non-first fragments aren't matched as flow heads). See [`examples/04_flow_heads.py`](https://github.com/stamparm/pcapy-ng/blob/master/examples/04_flow_heads.py). ### Address-set matching ### `addr_set` is a packed list of IPv4 addresses (4 bytes each, network order). Any packet whose source or destination is in it is delivered as `SET_MATCH`, regardless of port or protocol — a fast way to watch for traffic to/from a set of hosts (an allowlist, a blocklist, an asset inventory…). The set can be huge. **IPv4 only.** The classifier operates on IPv4; IPv6 packets are always `OTHER` and never match `addr_set`. (`setfilter("ip6 ...")` still works for plain IPv6 capture.) ```python import socket watch = ["198.51.100.10", "203.0.113.7"] addr_set = b"".join(socket.inet_aton(ip) for ip in watch) cap.loop_filtered(-1, on_match, (1 << 2), addr_set) # SET_MATCH only ``` ### `loop_to_buffer`: filling a shared buffer ### Same classification, but instead of calling Python per packet it writes the admitted packets **into a writable buffer you provide**, with the GIL released for the whole loop. Pair it with `multiprocessing.shared_memory` and a pool of workers when one thread isn't enough. ```python from multiprocessing import shared_memory shm = shared_memory.SharedMemory(create=True, size=64 * 1024 * 1024) res = cap.loop_to_buffer(-1, shm.buf, addr_set=addr_set, flow_cutoff=3) # res -> (written, dropped, overflow, bytes_used, ) ``` Each record is `[u32 caplen little-endian][u8 class][caplen bytes]`, written back-to-back with **no padding/alignment**. The only metadata stored is the caplen and the class — **no timestamp and no wire length**; if you need those, use `loop_filtered` (its callback gets the full `Pkthdr`). When the buffer fills, extra packets are counted in `overflow` instead of overrunning it. See [`examples/05_shared_memory_ring.py`](https://github.com/stamparm/pcapy-ng/blob/master/examples/05_shared_memory_ring.py). ### Live counters and non-Ethernet links ### `filtered_stats()` returns the current run's `(admitted, dropped, )` counters and is safe to read from another thread while the loop runs. `loop_filtered` needs to know where the IPv4 header starts, via `l2_offset`: `14` for Ethernet (default), `16` for Linux "cooked" capture (`DLT_LINUX_SLL`, the `any` device), `0` for raw IP (`DLT_RAW`). VLAN tags are skipped automatically. Derive it from `cap.datalink()`: ```python OFFSETS = {pcapy.DLT_EN10MB: 14, pcapy.DLT_LINUX_SLL: 16, pcapy.DLT_RAW: 0} cap.loop_filtered(-1, cb, l2_offset=OFFSETS.get(cap.datalink(), 14)) ``` --- ## Built-in security profile ## `loop_filtered(..., profile=1)` swaps the generic classifier for a built-in one aimed at security/traffic monitoring — it tags DNS, HTTP, TLS/QUIC handshakes, TCP SYNs and address-set hits in a single pass, configurable via `dns_port`, `dpi_ports` and `tls_ports`. It exists so tools like [Maltrail](https://github.com/stamparm/maltrail) can do their triage in C; if you're not building that kind of tool, ignore it and stay with the generic classes above. | index | `cls` id | name | meaning | |------:|---------:|------|---------| | 0 | `100` | DNS | UDP/TCP on `dns_port` | | 1 | `101` | HTTP | payload-bearing TCP on a `dpi_ports` port | | 2 | `102` | OTHER | none of the below (the shed-able bulk) | | 3 | `103` | ADDR | src/dst in `addr_set` (any protocol, incl. ICMP) | | 4 | `104` | HANDSHAKE | flow head on a `tls_ports` port (ClientHello / QUIC Initial) | | 5 | `105` | SYN | bare TCP SYN | Two numbers, two jobs — this is the part to get right: - The **`cls` id** delivered to your callback is `100 + index` (so it never collides with the generic `0`/`1`/`2`). - **`admit_mask` is a bitmask over the `index`, not the id.** To admit DNS + ADDR you write `(1 << 0) | (1 << 3)` — *not* `1 << 100`. The default mask for this profile is `59` (`= 1<<0 | 1<<1 | 1<<3 | 1<<4 | 1<<5`), i.e. everything except `OTHER` (index 2). The per-class counts in the return tuple are in `index` order (`admitted, dropped, dns, http, other, addr, handshake, syn`). --- ## `next_batch`: many packets per call ## `next_batch(max_n)` reads up to `max_n` packets in a single call and returns them as `(packet_bytes, packed_meta)`: `packet_bytes` is every packet concatenated, and `packed_meta` is an array of fixed 16-byte records — one per packet, four native-endian `uint32`s `(sec, usec, offset, caplen)` — where `offset`/`caplen` slice a packet out of `packet_bytes`. An empty `packed_meta` means EOF (offline) or timeout (live). It crosses the C↔Python boundary once per batch instead of once per packet, and unlike `loop_filtered` it does no classification — you get everything, with timestamps. ```python import struct pkts, meta = cap.next_batch(1024) for i in range(0, len(meta), 16): sec, usec, off, caplen = struct.unpack_from("=IIII", meta, i) data = pkts[off:off + caplen] ``` --- ## `set_fanout`: several handles on one interface ## One capture socket is one thread's worth of work. On Linux, `set_fanout(group_id, fanout_type)` joins a live, activated handle to a kernel `PACKET_FANOUT` group: open N handles on the same interface, put them all in the same group, read each in its own thread or process, and the kernel spreads the interface's packets across them without duplicating any. With the default `PACKET_FANOUT_HASH` every flow stays on one handle, so per-flow state stays consistent. ```python cap = pcapy.open_live(dev, 65535, True, 100) cap.set_fanout(0x4711, pcapy.PACKET_FANOUT_HASH) ``` Constants: `PACKET_FANOUT_HASH`, `PACKET_FANOUT_LB`, `PACKET_FANOUT_CPU`, `PACKET_FANOUT_ROLLOVER`, `PACKET_FANOUT_RND`. Linux only, live handles only; anything else raises. See [`examples/07_fanout_scale.py`](https://github.com/stamparm/pcapy-ng/blob/master/examples/07_fanout_scale.py). --- ## API reference ## **Module:** `open_live(dev, snaplen, promisc, timeout_ms)`, `open_offline(path)`, `create(dev)`, `findalldevs()`, `lookupdev()`, `compile(...)`, the `DLT_*`, `PCAP_D_*` and `PACKET_FANOUT_*` constants, and the `PcapError` / `BPFError` exceptions. **Reader (classic):** `next()` → `(hdr, data)`, `loop(cnt, cb)`, `dispatch(cnt, cb)`, `setfilter(bpf)`, `datalink()`, `getnet()`, `getmask()`, `getnonblock()/setnonblock()`, `setdirection(...)`, `stats()` → `(recv, drop, ifdrop)`, `dump_open(path)`, `sendpacket(data)`, `getfd()`, `close()`. Usable as a context manager. **Reader (unactivated, from `create()`):** `set_snaplen(n)`, `set_promisc(bool)`, `set_timeout(ms)`, `set_buffer_size(n)`, `set_rfmon(bool)` (monitor mode, where supported), then `activate()`. **Reader (filtering):** | Method | Purpose | |--------|---------| | `loop_filtered(cnt, cb, admit_mask=7, addr_set=b"", flow_cutoff=0, dpi_ports=None, tls_ports=None, dns_port=53, l2_offset=14, profile=0)` | classify in C; call `cb(hdr, data, cls)` only for admitted classes; return `(admitted, dropped, )` | | `loop_to_buffer(cnt, buf, ...same...)` | same, but write admitted packets into `buf` with the GIL released | | `filtered_stats()` | live counter snapshot of the current/last run | | `next_batch(max_n)` | read up to `max_n` packets in one call as `(packet_bytes, packed_meta)` | | `set_fanout(group_id, fanout_type=PACKET_FANOUT_HASH)` | Linux: join this live handle to a kernel `PACKET_FANOUT` group | `cnt = -1`/`0` means "until EOF (offline) or forever (live)". Code that must also run against stock Pcapy can probe with `hasattr(reader, "loop_filtered")` and fall back to `loop()` / `next()`. --- ## Examples ## Runnable, self-contained scripts in [`examples/`](https://github.com/stamparm/pcapy-ng/tree/master/examples) — each takes a device (live) or a `.pcap` (offline) as its first argument: | File | Shows | |------|-------| | `01_sniff_live.py` | classic live capture with a BPF filter | | `02_read_pcap.py` | read a `.pcap` and summarize it | | `03_filter_in_c.py` | `loop_filtered`: tag + count, drop `OTHER` in C | | `04_flow_heads.py` | grab the first packets of each connection (`flow_cutoff`) | | `05_shared_memory_ring.py` | `loop_to_buffer` producer + multiprocess consumers | | `06_live_stats.py` | poll `filtered_stats()` from another thread | | `07_fanout_scale.py` | `set_fanout`: several capture threads on one interface (Linux) | --- ## Building & compatibility ## Builds from source on CPython 3.9–3.14, on Linux, macOS and Windows (libpcap / Npcap); all it needs is a C++ compiler and the libpcap headers. CI builds and tests every commit on CPython 3.9–3.14 (Linux and macOS). Wheels are produced for those two platforms only, with [cibuildwheel](https://cibuildwheel.readthedocs.io/), and vendor libpcap, so installing from a wheel needs no compiler or `libpcap-dev`. **There are no Windows wheels** — on Windows pip builds from the sdist against the Npcap SDK. `set_fanout` is Linux-only; the classifier used by `loop_filtered` / `loop_to_buffer` is IPv4-only (IPv6 packets are always `OTHER`). Everything else is portable. --- ## License & credits ## Apache Software License — see [LICENSE](https://github.com/stamparm/pcapy-ng/blob/master/LICENSE). Pcapy-NG is maintained by Miroslav Stampar (`contact@sekuripy.hr`) and builds on the original Pcapy by CORE Security. Bug reports, patches and suggestions welcome. pcapy-ng-2.1.0/benchmarks/000077500000000000000000000000001524067405300153535ustar00rootroot00000000000000pcapy-ng-2.1.0/benchmarks/.gitignore000066400000000000000000000000061524067405300173370ustar00rootroot00000000000000data/ pcapy-ng-2.1.0/benchmarks/README.md000066400000000000000000000156051524067405300166410ustar00rootroot00000000000000# pcapy-ng benchmarks Measurement harness for the classic and the 2.0 capture APIs. It is deliberately separate from `tests/` — nothing here runs as part of the test suite, and nothing here is imported by the package. The point of this harness is to find out **where each API helps and where it does not**. Any scenario where a newer API is not faster is reported as such; see "Reading the results". ```sh python3 gen_pcap.py # once: build the deterministic capture (~175 MiB) python3 bench.py --reps 5 # run everything -> results/ python3 bench.py --only 'flowheads' # run a subset ``` `gen_pcap.py` and `bench.py` need a Python with `pcapy-ng` importable; `bench.py` re-executes itself with `sys.executable`, so run it with the interpreter you want to measure. ## Files | file | purpose | |---|---| | `gen_pcap.py` | deterministic capture generator + JSON manifest (ground truth) | | `pyequiv.py` | Python implementations of the same decisions the C classifier makes | | `bench.py` | the harness: scenarios, repetitions, metrics, CSV/JSON/Markdown output | | `fanout_live.py` | **optional**, Linux-only, live `PACKET_FANOUT` check (not reproducible) | | `data/` | generated capture + manifest (git-ignored: ~175 MiB, regenerate with `gen_pcap.py`) | | `results/` | `results.csv`, `results.json`, `SUMMARY.md` — **committed**, so every published number can be checked against the run that produced it | ## Methodology **Deterministic input.** `gen_pcap.py` is seeded, so the same arguments always produce the same capture. The default is 400 000 packets (~175 MiB, ~443 B average) of mixed TCP/UDP/ICMP traffic across ~6 500 flows (~23 600 directional 5-tuple keys), including ~3 % IPv6 and ~2 % VLAN-tagged frames so both the C classifier and the Python equivalents have to handle the non-IPv4 and tagged cases. **Ground truth.** The generator also writes a manifest containing, computed while the packets are being written: the exact number of packets each address set matches, and the exact number of flow heads for each cutoff. Every scenario asserts its admitted count against the manifest, so a "fast" result that silently admitted the wrong packets cannot pass unnoticed. **No disk in the measurement.** The capture is read once end to end before the first measurement to warm the page cache, and every repetition then re-reads the same file. The machine must have enough free RAM to keep the capture cached — check the `peak RSS` column and the reported free memory if numbers look unstable. **Process isolation.** Each measurement runs in a fresh subprocess (`bench.py --run-one`), so peak RSS, the C flow table and any large Python set are attributed to the scenario that allocated them and cannot leak into the next one. **What is timed.** Only the capture loop. Opening the reader, packing the address set, building the Python `set` and allocating the output buffer are setup and are reported separately as `setup_median_s` — relevant for the 1 000 000-address scenarios, where building the Python set is a real cost that the C path does not pay in the same form. **Repetitions.** `--reps N` (default 5) full rounds, scenario by scenario, so slow drift affects all scenarios similarly. The report gives median, min, max, stdev and the relative spread; treat differences smaller than the spread as noise. **Metrics.** Wall clock (`time.perf_counter`), process CPU time (`time.process_time`), packets processed per second, packets *delivered to Python* per second, admitted/dropped counts, overflow (for `loop_to_buffer`) and peak RSS (`ru_maxrss` of the child). "Delivered to Python" means per-packet Python objects were created: it is the whole capture for `next()`/`loop()`, the admitted subset for `loop_filtered()`, and zero for `loop_to_buffer()` and for the metadata-only `next_batch()` variants. ## Fairness rules 1. **Equal-work pairs.** Every `cls_*` / `buf_*` scenario has a `py_*` counterpart that reaches the same admitted set in Python on top of the classic API — parsing Ethernet/VLAN/IPv4, checking set membership or maintaining a flow dict. The harness prints both admitted counts and flags any difference. A speedup is only reported next to the counts that produced it. 2. **Raw overhead is labelled as such.** `next()`, `loop()` and `next_batch()` do different amounts of work per packet. They are in their own table, explicitly *not* an equal-work comparison; `next_batch` appears twice (metadata-only, and with a per-packet slice so it materialises the same objects `next()` does). 3. **The Python side is not sandbagged.** `pyequiv.py` uses precomputed `struct.Struct` objects, bound-method lookups hoisted out of the loop, integer sets and a plain dict. It is what a competent Python author would write, not a strawman — and not micro-tuned into unreadability either. 4. **Set size and key distribution are separated.** The address-set sweep runs twice: once padded with consecutive addresses (what expanding CIDR blocks gives you) and once with uniformly random ones. The C set is an open-addressing table whose probe behaviour depends on key distribution, so collapsing the two would attribute a hashing effect to set size. ## Known semantic difference (reported, not hidden) The C flow table is a fixed ~12 MB direct-mapped hash (1M slots) that **evicts on collision**; the Python equivalent uses an exact dict. When two live flows hash to the same slot they evict each other and their counters restart, so the C path reports a few percent *more* flow heads than exact per-tuple accounting. That is the documented memory/accuracy trade, so the flow-head scenarios assert "at least the exact count" and the report prints the actual delta. Every other scenario matches the manifest exactly. ## PACKET_FANOUT `set_fanout()` is a live, Linux-only kernel scaling feature: it cannot be measured from an offline capture, and its results depend on hardware, driver, IRQ affinity and whatever traffic happens to be on the wire. It is therefore **not** part of the reproducible benchmark. Use `fanout_live.py` separately, and treat its output as an observation about one machine at one moment, not as a portable number. ## Reading the results `results/SUMMARY.md` has four sections: raw API overhead, classification (equal work), `loop_to_buffer` (equal work), and a paired-speedup table. Things worth checking before quoting any number: * the `spread` column — if it is large, the machine was busy; * the admitted counts — a fast path that admits the wrong number of packets is a bug, not a win; * `setup_median_s` for the large address sets; * peak RSS for the buffer scenarios, which allocate the whole capture plus per-record headers. The committed `results/` is one machine at one moment — the exact CPU, kernel, Python and libpcap are in the Environment table at the top of `SUMMARY.md`, and every ratio quoted elsewhere in the project comes from that run. Re-running `bench.py` overwrites all three files. pcapy-ng-2.1.0/benchmarks/bench.py000066400000000000000000000673531524067405300170220ustar00rootroot00000000000000#!/usr/bin/env python3 """Reproducible benchmark harness for the pcapy-ng capture APIs. Measurement first: this harness does not try to make any API look good. Every scenario that makes a classification decision in C is paired with a Python implementation that makes the same logical decision on top of the classic API, and the harness asserts that both admit the same number of packets before it reports a speedup. Scenarios that only compare per-call overhead (next / loop / next_batch) are reported separately and are explicitly *not* equal-work comparisons. Usage: python3 gen_pcap.py # once: build the deterministic capture python3 bench.py --reps 5 # run everything, write results/ Each measurement runs in a fresh subprocess so that peak RSS, the C flow table and any large Python set are attributed to the scenario that allocated them. """ from __future__ import division, print_function import argparse import csv import json import os import platform import re import resource import statistics import struct import subprocess import sys import time HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) import pyequiv # noqa: E402 DEFAULT_PCAP = os.path.join(HERE, 'data', 'bench.pcap') RESULTS_DIR = os.path.join(HERE, 'results') OTHER, FLOW_HEAD, SET_MATCH = 0, 1, 2 M_ALL = (1 << OTHER) | (1 << FLOW_HEAD) | (1 << SET_MATCH) M_NONE = 0 M_SET = 1 << SET_MATCH M_HEAD = 1 << FLOW_HEAD # --------------------------------------------------------------------------------------- # # system information # --------------------------------------------------------------------------------------- # def _cpu_model(): try: with open('/proc/cpuinfo') as fh: for line in fh: if line.startswith('model name'): return line.split(':', 1)[1].strip() except IOError: pass return platform.processor() or 'unknown' def _libpcap_version(): """Ask the libpcap that pcapy actually loaded (may be a vendored copy in a wheel).""" import ctypes path = None try: with open('/proc/self/maps') as fh: for line in fh: m = re.search(r'(\S*libpcap[^\s]*)$', line.strip()) if m: path = m.group(1) break except IOError: pass try: lib = ctypes.CDLL(path) if path else ctypes.CDLL('libpcap.so.1') lib.pcap_lib_version.restype = ctypes.c_char_p return {'path': path, 'version': lib.pcap_lib_version().decode('utf-8', 'replace')} except Exception as exc: # pragma: no cover - diagnostics only return {'path': path, 'version': 'unknown (%s)' % exc} def _git_info(): try: out = subprocess.check_output(['git', '-C', HERE, 'rev-parse', '--short', 'HEAD'], stderr=subprocess.DEVNULL).decode().strip() dirty = subprocess.call(['git', '-C', HERE, 'diff', '--quiet', 'HEAD'], stderr=subprocess.DEVNULL) != 0 return out + ('-dirty' if dirty else '') except Exception: return 'unknown' def _cpu_freq_policy(): info = {} for name in ('scaling_governor', 'scaling_driver'): try: with open('/sys/devices/system/cpu/cpu0/cpufreq/' + name) as fh: info[name] = fh.read().strip() except IOError: pass try: with open('/sys/devices/system/cpu/cpu0/cpufreq/scaling_max_freq') as fh: info['scaling_max_freq_khz'] = int(fh.read().strip()) except IOError: pass return info def system_info(manifest): import pcapy try: import importlib.metadata as md pcapy_version = md.version('pcapy-ng') except Exception: pcapy_version = 'unknown' return { 'cpu': _cpu_model(), 'cpu_count': os.cpu_count(), 'cpu_freq_policy': _cpu_freq_policy(), 'loadavg_at_start': os.getloadavg(), 'platform': platform.platform(), 'kernel': platform.release(), 'python': sys.version.split()[0], 'python_implementation': platform.python_implementation(), 'python_executable': sys.executable, 'pcapy_ng_version': pcapy_version, 'pcapy_module': getattr(pcapy, '__file__', 'unknown'), 'pcapy_ng_git': _git_info(), 'libpcap': _libpcap_version(), 'capture': { 'path': manifest['path'], 'packets': manifest['packets'], 'bytes': manifest['bytes'], 'file_size': manifest['file_size'], 'avg_packet_size': round(manifest['avg_packet_size'], 1), 'ipv4_packets': manifest['ipv4_packets'], 'ipv6_packets': manifest['ipv6_packets'], 'vlan_packets': manifest['vlan_packets'], 'distinct_flow_keys': manifest['distinct_flow_keys'], }, } # --------------------------------------------------------------------------------------- # # scenarios # --------------------------------------------------------------------------------------- # def build_scenarios(manifest): """Return an ordered list of scenario dicts. group 'raw' per-call overhead only; NOT an equal-work comparison 'classify' C classifier vs Python doing the same logical decision 'buffer' loop_to_buffer vs Python copying into a bytearray pair name of the counterpart scenario (equal work), or None """ sets = manifest['addr_sets'] sizes = manifest['addr_set_sizes'] total = manifest['packets'] heads = manifest['flow_head_counts'] s = [] # --- raw API overhead --------------------------------------------------------------- s.append(dict(name='raw_next', group='raw', kind='classic_next', params={}, expect=total, note='one Python call + 2 objects per packet')) s.append(dict(name='raw_loop', group='raw', kind='classic_loop', params={}, expect=total, note='one callback + 2 objects per packet')) for bs in (32, 128, 512, 1024): s.append(dict(name='raw_next_batch_%d_meta' % bs, group='raw', kind='next_batch', params={'batch': bs, 'slice': False}, expect=total, note='metadata parsed, packet bytes not sliced out')) s.append(dict(name='raw_next_batch_%d_slice' % bs, group='raw', kind='next_batch', params={'batch': bs, 'slice': True}, expect=total, note='one bytes object per packet, like next()')) # --- classifier: admit rate --------------------------------------------------------- s.append(dict(name='cls_admit_all', group='classify', kind='filtered', params={'mask': M_ALL}, expect=total, pair='py_admit_all')) s.append(dict(name='py_admit_all', group='classify', kind='py_admit_all', params={}, expect=total)) for tag, key in (('50', '50pct'), ('10', '10pct'), ('1', '1pct'), ('0', '0pct')): exp = sets[key]['expected_matches'] s.append(dict(name='cls_addrset_%spct' % tag, group='classify', kind='filtered', params={'mask': M_SET, 'set': key}, expect=exp, pair='py_addrset_%spct' % tag, note='%.2f%% of packets admitted' % (100.0 * exp / total))) s.append(dict(name='py_addrset_%spct' % tag, group='classify', kind='py_addrset', params={'set': key}, expect=exp)) s.append(dict(name='cls_admit_none', group='classify', kind='filtered', params={'mask': M_NONE}, expect=0, pair='py_admit_none', note='classify everything, deliver nothing')) s.append(dict(name='py_admit_none', group='classify', kind='py_admit_none', params={}, expect=0)) # --- classifier: flow heads --------------------------------------------------------- # The C flow table is direct-mapped with eviction (fixed ~12 MB, 1M slots), so two flows # that hash to the same slot evict each other and their counters restart: the C path # admits a few percent MORE flow heads than exact per-tuple accounting. That is a design # trade (bounded memory) and is reported, not hidden -- hence expect_mode='min'. for cutoff in (1, 3): s.append(dict(name='cls_flowheads_%d' % cutoff, group='classify', kind='filtered', params={'mask': M_HEAD, 'cutoff': cutoff}, expect=heads[str(cutoff)], expect_mode='min', pair='py_flowheads_%d' % cutoff, note='C table is direct-mapped with eviction; admits >= exact count')) s.append(dict(name='py_flowheads_%d' % cutoff, group='classify', kind='py_flowheads', params={'cutoff': cutoff}, expect=heads[str(cutoff)])) # --- classifier: address-set size sweep (match rate held constant) ------------------- # Run twice: once with consecutive padding addresses (what expanding CIDR blocks gives # you) and once with uniformly random ones, because set size and key distribution are # separate variables for an open-addressing hash. for dist, key in (('seq', 'sizeset'), ('rand', 'sizeset_rand')): table = sizes if dist == 'seq' else manifest['addr_set_sizes_random'] for size in ('10', '1000', '100000', '1000000'): exp = table[size]['expected_matches'] s.append(dict(name='cls_setsize_%s_%s' % (dist, size), group='classify', kind='filtered', params={'mask': M_SET, key: size}, expect=exp, pair='py_setsize_%s_%s' % (dist, size), note='%s addresses, %s padding, %.2f%% admitted' % (size, 'consecutive' if dist == 'seq' else 'random', 100.0 * exp / total))) s.append(dict(name='py_setsize_%s_%s' % (dist, size), group='classify', kind='py_setsize', params={key: size}, expect=exp)) # --- loop_to_buffer ----------------------------------------------------------------- s.append(dict(name='buf_admit_all', group='buffer', kind='to_buffer', params={'mask': M_ALL}, expect=total, pair='py_buf_admit_all')) s.append(dict(name='py_buf_admit_all', group='buffer', kind='py_buf_all', params={}, expect=total)) exp10 = sets['10pct']['expected_matches'] s.append(dict(name='buf_addrset_10pct', group='buffer', kind='to_buffer', params={'mask': M_SET, 'set': '10pct'}, expect=exp10, pair='py_buf_addrset_10pct')) s.append(dict(name='py_buf_addrset_10pct', group='buffer', kind='py_buf_addrset', params={'set': '10pct'}, expect=exp10)) s.append(dict(name='buf_flowheads_3', group='buffer', kind='to_buffer', params={'mask': M_HEAD, 'cutoff': 3}, expect=heads['3'], expect_mode='min', pair='py_buf_flowheads_3', note='C table is direct-mapped with eviction; admits >= exact count')) s.append(dict(name='py_buf_flowheads_3', group='buffer', kind='py_buf_flowheads', params={'cutoff': 3}, expect=heads['3'])) return s # --------------------------------------------------------------------------------------- # # child: run exactly one measurement and print JSON # --------------------------------------------------------------------------------------- # def _addr_bytes(ips): return b''.join(struct.pack('!I', ip) for ip in ips) def _noop(hdr, data, cls): return None def _noop2(hdr, data): return None def run_one(scenario, manifest): import pcapy path = manifest['path'] p = scenario['params'] kind = scenario['kind'] # ---- setup (not measured; reported separately) ---- t_setup = time.perf_counter() addr_b = b'' addr_py = None buf = None if 'set' in p: ips = manifest['addr_sets'][p['set']]['ips'] addr_b, addr_py = _addr_bytes(ips), set(ips) if 'sizeset' in p: ips = manifest['addr_set_sizes'][p['sizeset']]['ips'] addr_b, addr_py = _addr_bytes(ips), set(ips) if 'sizeset_rand' in p: ips = manifest['addr_set_sizes_random'][p['sizeset_rand']]['ips'] addr_b, addr_py = _addr_bytes(ips), set(ips) if kind in ('to_buffer', 'py_buf_all', 'py_buf_addrset', 'py_buf_flowheads'): buf = bytearray(manifest['bytes'] + 5 * manifest['packets'] + 4096) cutoff = p.get('cutoff', 0) mask = p.get('mask', M_ALL) reader = pcapy.open_offline(path) setup_s = time.perf_counter() - t_setup # ---- measured region ---- cpu0 = time.process_time() t0 = time.perf_counter() if kind == 'classic_next': n = 0 while True: hdr, data = reader.next() if hdr is None: break n += 1 res = {'processed': n, 'admitted': n, 'delivered': n, 'dropped': 0} elif kind == 'classic_loop': # loop() returns no count, so the callback is the same no-op the loop_filtered # scenarios use and the packet count comes from the manifest (asserted afterwards). reader.loop(-1, _noop2) res = {'processed': manifest['packets'], 'admitted': manifest['packets'], 'delivered': manifest['packets'], 'dropped': 0} elif kind == 'next_batch': bs = p['batch'] do_slice = p['slice'] n = 0 while True: pkts, meta = reader.next_batch(bs) if not meta: break for off in range(0, len(meta), 16): sec, usec, o, caplen = struct.unpack_from('=IIII', meta, off) if do_slice: _ = pkts[o:o + caplen] n += 1 res = {'processed': n, 'admitted': n, 'delivered': n if do_slice else 0, 'dropped': 0} elif kind == 'filtered': r = reader.loop_filtered(-1, _noop, mask, addr_b, cutoff) res = {'processed': r[0] + r[1], 'admitted': r[0], 'delivered': r[0], 'dropped': r[1]} elif kind == 'to_buffer': r = reader.loop_to_buffer(-1, buf, mask, addr_b, cutoff) res = {'processed': r[0] + r[1], 'admitted': r[0], 'delivered': 0, 'dropped': r[1], 'overflow': r[2], 'bytes_used': r[3]} elif kind == 'py_admit_all': res = pyequiv.classify_all(reader) elif kind == 'py_admit_none': res = pyequiv.classify_none(reader) elif kind in ('py_addrset', 'py_setsize'): res = pyequiv.addr_set_match(reader, addr_py) elif kind == 'py_flowheads': res = pyequiv.flow_heads(reader, cutoff) elif kind == 'py_buf_all': res = pyequiv.buffer_all(reader, buf) elif kind == 'py_buf_addrset': res = pyequiv.buffer_addr_set(reader, buf, addr_py) elif kind == 'py_buf_flowheads': res = pyequiv.buffer_flow_heads(reader, buf, cutoff) else: raise ValueError('unknown kind %r' % kind) wall = time.perf_counter() - t0 cpu = time.process_time() - cpu0 res.setdefault('dropped', res['processed'] - res['admitted']) res.update({ 'name': scenario['name'], 'wall_s': wall, 'cpu_s': cpu, 'setup_s': setup_s, 'peak_rss_kib': resource.getrusage(resource.RUSAGE_SELF).ru_maxrss, }) return res # --------------------------------------------------------------------------------------- # # parent: orchestration, aggregation, reporting # --------------------------------------------------------------------------------------- # def spawn(scenario, manifest_path, python=None): payload = json.dumps({'scenario': scenario, 'manifest': manifest_path}) out = subprocess.check_output([python or sys.executable, os.path.abspath(__file__), '--run-one', payload]) return json.loads(out.decode().strip().splitlines()[-1]) def aggregate(samples): def stat(key): vals = sorted(x[key] for x in samples) med = statistics.median(vals) return { 'median': med, 'min': vals[0], 'max': vals[-1], 'mean': statistics.mean(vals), 'stdev': statistics.stdev(vals) if len(vals) > 1 else 0.0, 'rel_spread_pct': 100.0 * (vals[-1] - vals[0]) / med if med else 0.0, } first = samples[0] return { 'name': first['name'], 'reps': len(samples), 'wall': stat('wall_s'), 'cpu': stat('cpu_s'), 'setup_s_median': statistics.median(x['setup_s'] for x in samples), 'peak_rss_kib_max': max(x['peak_rss_kib'] for x in samples), 'processed': first['processed'], 'admitted': first['admitted'], 'dropped': first['dropped'], 'delivered': first['delivered'], 'overflow': first.get('overflow', 0), 'samples_wall_s': [x['wall_s'] for x in samples], } def fmt_rate(n, seconds): return n / seconds if seconds > 0 else float('inf') def write_reports(sysinfo, scenarios, agg, outdir): if not os.path.isdir(outdir): os.makedirs(outdir) by_name = {a['name']: a for a in agg} scen_by_name = {s['name']: s for s in scenarios} total = sysinfo['capture']['packets'] rows = [] for a in agg: sc = scen_by_name[a['name']] w = a['wall']['median'] exp = sc['expect'] mode = sc.get('expect_mode', 'exact') ok = (a['admitted'] == exp) if mode == 'exact' else (a['admitted'] >= exp) rows.append({ 'scenario': a['name'], 'group': sc['group'], 'reps': a['reps'], 'wall_median_s': round(w, 6), 'wall_min_s': round(a['wall']['min'], 6), 'wall_max_s': round(a['wall']['max'], 6), 'wall_stdev_s': round(a['wall']['stdev'], 6), 'wall_spread_pct': round(a['wall']['rel_spread_pct'], 2), 'cpu_median_s': round(a['cpu']['median'], 6), 'setup_median_s': round(a['setup_s_median'], 6), 'pkts_processed_per_s': int(fmt_rate(a['processed'], w)), 'pkts_delivered_per_s': int(fmt_rate(a['delivered'], w)), 'processed': a['processed'], 'admitted': a['admitted'], 'dropped': a['dropped'], 'delivered': a['delivered'], 'overflow': a['overflow'], 'peak_rss_mib': round(a['peak_rss_kib_max'] / 1024.0, 1), 'expected_admitted': exp, 'expect_mode': mode, 'admitted_delta_pct': round(100.0 * (a['admitted'] - exp) / exp, 3) if exp else 0.0, 'admitted_ok': ok, 'pair': sc.get('pair', ''), 'note': sc.get('note', ''), }) with open(os.path.join(outdir, 'results.csv'), 'w') as fh: writer = csv.DictWriter(fh, fieldnames=list(rows[0].keys())) writer.writeheader() writer.writerows(rows) with open(os.path.join(outdir, 'results.json'), 'w') as fh: json.dump({'system': sysinfo, 'scenarios': scenarios, 'results': agg, 'table': rows}, fh, indent=2, sort_keys=True) # --- markdown summary --- L = [] A = L.append cap = sysinfo['capture'] A('# pcapy-ng benchmark results\n') A('Offline capture, deterministic input, %d repetitions per scenario, medians reported.\n' % agg[0]['reps']) A('## Environment\n') A('| item | value |') A('|---|---|') A('| CPU | %s (%d logical CPUs) |' % (sysinfo['cpu'], sysinfo['cpu_count'])) A('| CPU freq policy | %s |' % (sysinfo['cpu_freq_policy'] or 'n/a')) A('| OS / kernel | %s |' % sysinfo['platform']) A('| Python | %s (%s) |' % (sysinfo['python'], sysinfo['python_implementation'])) A('| libpcap | %s |' % sysinfo['libpcap']['version']) A('| libpcap path | `%s` |' % sysinfo['libpcap']['path']) A('| pcapy-ng | %s (git %s) |' % (sysinfo['pcapy_ng_version'], sysinfo['pcapy_ng_git'])) A('| pcapy module | `%s` |' % sysinfo['pcapy_module']) A('| load average at start | %.2f %.2f %.2f |' % tuple(sysinfo['loadavg_at_start'])) A('| capture | %d packets, %.1f MiB, avg %.0f B/packet |' % (cap['packets'], cap['file_size'] / 1048576.0, cap['avg_packet_size'])) A('| capture mix | %d IPv4, %d IPv6, %d VLAN-tagged, %d distinct flow keys |' % (cap['ipv4_packets'], cap['ipv6_packets'], cap['vlan_packets'], cap['distinct_flow_keys'])) A('') def table(group, title, blurb): A('## %s\n' % title) A(blurb + '\n') A('| scenario | wall median (s) | spread | processed/s | delivered to Python/s | ' 'CPU (s) | peak RSS (MiB) | admitted | dropped |') A('|---|---:|---:|---:|---:|---:|---:|---:|---:|') for r in rows: if r['group'] != group: continue A('| `%s` | %.4f | ±%.1f%% | %s | %s | %.3f | %.1f | %d | %d |' % ( r['scenario'], r['wall_median_s'], r['wall_spread_pct'] / 2.0, '{:,}'.format(r['pkts_processed_per_s']), '{:,}'.format(r['pkts_delivered_per_s']), r['cpu_median_s'], r['peak_rss_mib'], r['admitted'], r['dropped'])) A('') table('raw', 'Raw API overhead (NOT equal work)', 'These three APIs do different amounts of work per packet, so this table is about ' 'per-call overhead only, not about who is "faster" at the same job. `next()` and ' '`loop()` build a header object and a bytes object per packet; `next_batch(..._meta)` ' 'builds two objects per *batch* and only parses the metadata; `next_batch(..._slice)` ' 'adds one bytes object per packet so it delivers the same thing `next()` does.') table('classify', 'Classification (equal work: C classifier vs Python equivalent)', 'Each `cls_*` row has a `py_*` counterpart that reaches the same admitted set in ' 'Python on top of the classic API. The harness asserts the admitted counts match ' 'before any comparison is drawn.') table('buffer', 'loop_to_buffer (equal work: C vs Python copying into a bytearray)', '`loop_to_buffer` writes `[u32 caplen][u8 class][packet]` records with the GIL ' 'released; the Python equivalent produces the same records in a preallocated ' '`bytearray` from the classic API.') A('## Paired speedups\n') A('Ratio of Python-equivalent wall time to C wall time, for pairs that admit the same ' 'packets. Values below 1.0 mean the C path is *slower* and are reported as such.\n') A('| C scenario | Python equivalent | admitted (C / Python) | C wall (s) | Python wall (s) ' '| speedup |') A('|---|---|---:|---:|---:|---:|') for r in rows: pair = r['pair'] if not pair or pair not in by_name: continue ca, pa = by_name[r['scenario']], by_name[pair] c, py = ca['wall']['median'], pa['wall']['median'] counts = '{:,}'.format(ca['admitted']) if ca['admitted'] != pa['admitted']: counts += ' / {:,} ({:+.2f}%)'.format( pa['admitted'], 100.0 * (ca['admitted'] - pa['admitted']) / pa['admitted']) A('| `%s` | `%s` | %s | %.4f | %.4f | **%.2fx** |' % (r['scenario'], pair, counts, c, py, py / c)) A('') A('Where the two admitted counts differ, the reason is the C flow table: it is ' 'direct-mapped with eviction, so colliding flows restart their counters and a few ' 'percent extra packets are reported as flow heads. The Python equivalent uses an exact ' 'dict and is the reference count.\n') # Structural honesty check: list every pair where the newer API is not a clear win, so a # regression or a bad case cannot be quietly buried in the table above. weak = [] for r in rows: pair = r['pair'] if not pair or pair not in by_name: continue c, py = by_name[r['scenario']]['wall']['median'], by_name[pair]['wall']['median'] if py / c < 1.2: weak.append((r['scenario'], pair, py / c, r['note'])) A('## Where the newer API does NOT help\n') if weak: A('Pairs where the C path is less than 1.2x faster than plain Python (or slower):\n') A('| C scenario | speedup vs Python | note |') A('|---|---:|---|') for name, pair, ratio, note in weak: A('| `%s` | **%.2fx** | %s |' % (name, ratio, note or '')) else: A('None: every paired scenario was at least 1.2x faster in C. (This section exists so ' 'that regressions and bad cases show up on their own, not only if someone reads the ' 'full table.)') A('') bad = [r for r in rows if not r['admitted_ok']] A('## Correctness cross-check\n') if bad: A('**%d scenario(s) did not admit the expected number of packets:**\n' % len(bad)) for r in bad: A('* `%s`: admitted %d, expected %s%d (%+.2f%%)' % (r['scenario'], r['admitted'], '>= ' if r['expect_mode'] == 'min' else '', r['expected_admitted'], r['admitted_delta_pct'])) else: A('All %d scenarios admitted the number of packets predicted by the capture manifest ' '(exact match, except the flow-head scenarios where the C table is allowed to admit ' 'more; their actual deltas are in the paired table above).' % len(rows)) A('') with open(os.path.join(outdir, 'SUMMARY.md'), 'w') as fh: fh.write('\n'.join(L)) return rows def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--run-one', help=argparse.SUPPRESS) ap.add_argument('--pcap', default=DEFAULT_PCAP) ap.add_argument('--reps', type=int, default=5) ap.add_argument('--out', default=RESULTS_DIR) ap.add_argument('--only', help='regex: run only matching scenarios') ap.add_argument('--report-only', action='store_true', help='regenerate CSV/Markdown from an existing results.json, no measuring') args = ap.parse_args() if args.report_only: with open(os.path.join(args.out, 'results.json')) as fh: saved = json.load(fh) write_reports(saved['system'], saved['scenarios'], saved['results'], args.out) print('regenerated reports in %s' % args.out, file=sys.stderr) return if args.run_one: req = json.loads(args.run_one) with open(req['manifest']) as fh: manifest = json.load(fh) print(json.dumps(run_one(req['scenario'], manifest))) return manifest_path = args.pcap + '.manifest.json' if not os.path.exists(manifest_path): sys.exit('missing %s -- run gen_pcap.py first' % manifest_path) with open(manifest_path) as fh: manifest = json.load(fh) # Warm the page cache so the benchmark measures the API, not the disk. with open(args.pcap, 'rb') as fh: while fh.read(1 << 22): pass scenarios = build_scenarios(manifest) if args.only: rx = re.compile(args.only) scenarios = [s for s in scenarios if rx.search(s['name'])] sysinfo = system_info(manifest) print('%d scenarios x %d reps' % (len(scenarios), args.reps), file=sys.stderr) samples = {s['name']: [] for s in scenarios} for rep in range(args.reps): for s in scenarios: r = spawn(s, manifest_path) samples[s['name']].append(r) print(' rep %d/%d %-26s %8.4f s admitted=%-8d rss=%dMiB' % (rep + 1, args.reps, s['name'], r['wall_s'], r['admitted'], r['peak_rss_kib'] // 1024), file=sys.stderr) agg = [aggregate(samples[s['name']]) for s in scenarios] rows = write_reports(sysinfo, scenarios, agg, args.out) print('\nwrote %s/{results.csv,results.json,SUMMARY.md}' % args.out, file=sys.stderr) bad = [r for r in rows if not r['admitted_ok']] if bad: print('WARNING: %d scenario(s) admitted an unexpected count' % len(bad), file=sys.stderr) if __name__ == '__main__': main() pcapy-ng-2.1.0/benchmarks/fanout_live.py000066400000000000000000000107111524067405300202400ustar00rootroot00000000000000#!/usr/bin/env python3 """OPTIONAL live PACKET_FANOUT scaling check -- Linux only, results are hardware dependent. This is deliberately NOT part of the reproducible offline benchmark. It measures a kernel feature on live traffic, so the numbers depend on the NIC, the driver, RSS/IRQ affinity, the traffic actually on the wire during the run and what else the machine is doing. Two runs on the same box can differ substantially; runs on different boxes are not comparable at all. What it does: opens N live capture handles on one interface, joins them all to the same PACKET_FANOUT group, reads each in its own thread for a fixed duration, and reports per-handle and total packet counts plus libpcap's own drop counters. Compare N=1 against N=2/4/8 to see whether fanout is buying anything on YOUR hardware with YOUR traffic. sudo python3 fanout_live.py eth0 --handles 1,2,4 --seconds 10 Needs root or CAP_NET_RAW. Generate load separately (e.g. iperf3) if the link is idle -- an idle link measures nothing. """ from __future__ import division, print_function import argparse import json import platform import sys import threading import time try: import pcapy except ImportError: sys.exit('pcapy-ng is not installed') SNAPLEN = 65535 TIMEOUT_MS = 100 def capture_worker(dev, group, seconds, idx, results, use_fanout, barrier): cap = pcapy.open_live(dev, SNAPLEN, True, TIMEOUT_MS) if use_fanout: cap.set_fanout(group, pcapy.PACKET_FANOUT_HASH) barrier.wait() n = 0 deadline = time.time() + seconds while time.time() < deadline: try: hdr, _ = cap.next() except pcapy.PcapError: continue if hdr is not None: n += 1 try: recv, drop, ifdrop = cap.stats() except Exception: recv = drop = ifdrop = -1 results[idx] = {'packets': n, 'pcap_recv': recv, 'pcap_drop': drop, 'pcap_ifdrop': ifdrop} cap.close() def run(dev, handles, seconds, group): use_fanout = handles > 1 results = [None] * handles barrier = threading.Barrier(handles + 1) threads = [threading.Thread(target=capture_worker, args=(dev, group, seconds, i, results, use_fanout, barrier)) for i in range(handles)] for t in threads: t.start() barrier.wait() t0 = time.time() for t in threads: t.join() wall = time.time() - t0 total = sum(r['packets'] for r in results) return { 'handles': handles, 'fanout': use_fanout, 'seconds': round(wall, 3), 'total_packets': total, 'packets_per_s': int(total / wall) if wall else 0, 'per_handle': results, 'balance_min_max': (min(r['packets'] for r in results), max(r['packets'] for r in results)), } def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('device') ap.add_argument('--handles', default='1,2,4', help='comma-separated handle counts to try (default 1,2,4)') ap.add_argument('--seconds', type=float, default=10.0) ap.add_argument('--group', type=lambda x: int(x, 0), default=0x4711) ap.add_argument('--json', help='write results to this file') args = ap.parse_args() if platform.system() != 'Linux': sys.exit('PACKET_FANOUT is Linux-only') out = {'device': args.device, 'kernel': platform.release(), 'runs': [], 'warning': 'live capture: hardware, driver, IRQ affinity and the traffic present ' 'during the run all affect these numbers; not reproducible across ' 'machines or runs'} for h in [int(x) for x in args.handles.split(',')]: print('capturing %.0fs with %d handle(s)%s ...' % (args.seconds, h, ' (fanout)' if h > 1 else ''), file=sys.stderr) r = run(args.device, h, args.seconds, args.group) out['runs'].append(r) print(' %d packets %d pkt/s per-handle min/max %s' % (r['total_packets'], r['packets_per_s'], r['balance_min_max']), file=sys.stderr) base = out['runs'][0]['packets_per_s'] or 1 for r in out['runs']: r['scaling_vs_first'] = round(r['packets_per_s'] / base, 3) text = json.dumps(out, indent=2, sort_keys=True) if args.json: with open(args.json, 'w') as fh: fh.write(text + '\n') print(text) if __name__ == '__main__': main() pcapy-ng-2.1.0/benchmarks/gen_pcap.py000066400000000000000000000306411524067405300175050ustar00rootroot00000000000000#!/usr/bin/env python3 """Deterministic synthetic capture generator for the pcapy-ng benchmarks. Writes a pcap file plus a JSON manifest describing exactly what is in it: packet and byte counts, the address sets used by the addr_set benchmarks (with their exact expected match counts) and the exact expected FLOW_HEAD counts per flow cutoff. The harness uses the manifest as ground truth, so every benchmark can assert that the C classifier and its Python equivalent admitted the same packets. Everything is driven by a fixed seed, so the same arguments always produce byte-identical output. python3 gen_pcap.py --packets 400000 --out data/bench.pcap """ import argparse import json import os import random import struct import sys SEED = 20260815 ETH_P_IP = 0x0800 ETH_P_IPV6 = 0x86DD ETH_P_VLAN = 0x8100 # Traffic mix (fractions of generated flows / packets) FRAC_IPV6 = 0.03 # IPv6 packets: always OTHER for the IPv4 classifier FRAC_VLAN = 0.02 # VLAN-tagged frames: both C and Python must skip the tag def ip2int(s): a, b, c, d = (int(x) for x in s.split('.')) return (a << 24) | (b << 16) | (c << 8) | d def int2bytes(v): return struct.pack('!I', v) def checksum(data): if len(data) % 2: data += b'\x00' s = sum(struct.unpack('!%dH' % (len(data) // 2), data)) while s >> 16: s = (s & 0xFFFF) + (s >> 16) return (~s) & 0xFFFF def eth(ethertype, vlan=False): hdr = b'\x02\x00\x00\x00\x00\x01' + b'\x02\x00\x00\x00\x00\x02' if vlan: return hdr + struct.pack('!HHH', ETH_P_VLAN, 0x0064, ethertype) return hdr + struct.pack('!H', ethertype) def ipv4(proto, src, dst, payload): total = 20 + len(payload) hdr = struct.pack('!BBHHHBBH4s4s', 0x45, 0, total, 0x1234, 0x4000, 64, proto, 0, int2bytes(src), int2bytes(dst)) hdr = hdr[:10] + struct.pack('!H', checksum(hdr)) + hdr[12:] return hdr + payload def tcp(sport, dport, flags, payload, seq=1): hdr = struct.pack('!HHIIBBHHH', sport, dport, seq, 0, 0x50, flags, 8192, 0, 0) return hdr + payload def udp(sport, dport, payload): return struct.pack('!HHHH', sport, dport, 8 + len(payload), 0) + payload def icmp(payload): return struct.pack('!BBHHH', 8, 0, 0, 1, 1) + payload def ipv6_packet(rnd, payload_len): src = bytes(rnd.randrange(256) for _ in range(16)) dst = bytes(rnd.randrange(256) for _ in range(16)) payload = bytes(payload_len) hdr = struct.pack('!IHBB', (6 << 28), len(payload), 17, 64) + src + dst return hdr + payload class PcapWriter(object): """Minimal little-endian pcap writer (linktype EN10MB, microsecond resolution).""" def __init__(self, path, snaplen=65535): self.fh = open(path, 'wb') self.fh.write(struct.pack('= 0.97, }) total += count return flows, externals def generate(path, n_packets): rnd = random.Random(SEED) flows, externals = build_flows(rnd, n_packets) # Interleave the flows: emit packets in a shuffled order, which preserves per-flow # ordering statistics while producing a realistically mixed capture. order = [] for idx, f in enumerate(flows): order.extend([idx] * f['count']) rnd.shuffle(order) writer = PcapWriter(path) ext_counts = {} # external IP -> number of packets it appears in flow_seen = {} # directional 5-tuple -> packets seen so far head_counts = {1: 0, 2: 0, 3: 0, 5: 0} n_ipv4 = n_ipv6 = n_vlan = 0 emitted = [0] * len(flows) ts_sec, ts_usec = 1767225600, 0 for idx in order: f = flows[idx] i = emitted[idx] emitted[idx] += 1 ts_usec += 7 if ts_usec >= 1000000: ts_usec -= 1000000 ts_sec += 1 vlan = rnd.random() < FRAC_VLAN if rnd.random() < FRAC_IPV6: frame = eth(ETH_P_IPV6, vlan) + ipv6_packet(rnd, rnd.randrange(40, 200)) writer.write(ts_sec, ts_usec, frame) n_ipv6 += 1 n_vlan += 1 if vlan else 0 continue outbound = (i % 2 == 0) src = f['internal'] if outbound else f['external'] dst = f['external'] if outbound else f['internal'] sport = f['sport'] if outbound else f['dport'] dport = f['dport'] if outbound else f['sport'] if f['bulk'] and i > 3: plen = rnd.choice([1400, 1400, 512, 128]) # bulk transfer, mixed sizes elif f['proto'] == 6 and i < 3: plen = 0 # handshake else: plen = rnd.choice([0, 0, 24, 64, 120, 300, rnd.randrange(0, 700)]) payload = bytes(plen) if f['proto'] == 6: flags = 0x02 if i == 0 else (0x12 if i == 1 else (0x18 if plen else 0x10)) l4 = tcp(sport, dport, flags, payload, seq=i + 1) elif f['proto'] == 17: l4 = udp(sport, dport, payload) else: l4 = icmp(payload) frame = eth(ETH_P_IP, vlan) + ipv4(f['proto'], src, dst, l4) writer.write(ts_sec, ts_usec, frame) n_ipv4 += 1 n_vlan += 1 if vlan else 0 ext_counts[f['external']] = ext_counts.get(f['external'], 0) + 1 # The C classifier only tracks flow heads for TCP/UDP (it needs ports for the key), # so the ground truth here must do the same. if f['proto'] in (6, 17): key = (src, dst, sport, dport, f['proto']) seen = flow_seen.get(key, 0) + 1 flow_seen[key] = seen for cutoff in head_counts: if seen <= cutoff: head_counts[cutoff] += 1 writer.close() # --- address sets with exact expected match counts ----------------------------------- ranked = sorted(ext_counts.items(), key=lambda kv: (-kv[1], kv[0])) def set_for_fraction(target): """Pick external IPs (largest first) until their packets reach ~target of the file.""" want = int(round(target * writer.packets)) chosen, got = [], 0 for ip, c in ranked: if got >= want: break chosen.append(ip) got += c return chosen, got sets = {} for name, target in (('50pct', 0.50), ('10pct', 0.10), ('1pct', 0.01)): ips, matched = set_for_fraction(target) sets[name] = {'ips': ips, 'expected_matches': matched, 'actual_fraction': matched / float(writer.packets)} # A set that matches nothing. Its addresses are drawn at random (not sequentially) so # this scenario measures "nothing admitted", not hash-clustering behaviour -- the latter # gets its own sweep below. absent = random.Random(SEED + 1) present = set(ext_counts) zero_ips = [] while len(zero_ips) < 1000: v = absent.randrange(1, 0xFFFFFFFF) if v not in present: zero_ips.append(v) sets['0pct'] = {'ips': zero_ips, 'expected_matches': 0, 'actual_fraction': 0.0} # Set-size sweep: every set contains the SAME small core of matching addresses and is # padded to size with addresses that never appear in the capture, so the match rate is # constant and the only variable is how many addresses the set holds. # # Two paddings, because set size and key distribution are different variables: # 'seq' consecutive addresses (what you get from expanding CIDR blocks) -- the worst # case for a multiplicative hash whose masked bits come from the low octets # 'random' uniformly random addresses -- the average case core = [ip for ip, _ in ranked[:10]] core_matches = sum(ext_counts[ip] for ip in core) pad_base = ip2int('198.18.0.0') rnd_pad = random.Random(SEED + 2) sizes = (10, 1000, 100000, 1000000) size_sets = {} for size in sizes: ips = list(core) + [pad_base + i + 1 for i in range(max(0, size - len(core)))] size_sets[str(size)] = {'ips': ips, 'expected_matches': core_matches, 'actual_fraction': core_matches / float(writer.packets)} pad_random = [] seen_pad = set(ext_counts) while len(pad_random) < max(sizes): v = rnd_pad.randrange(1, 0xFFFFFFFF) if v not in seen_pad: seen_pad.add(v) pad_random.append(v) size_sets_random = {} for size in sizes: ips = list(core) + pad_random[:max(0, size - len(core))] size_sets_random[str(size)] = {'ips': ips, 'expected_matches': core_matches, 'actual_fraction': core_matches / float(writer.packets)} manifest = { 'seed': SEED, 'path': os.path.abspath(path), 'packets': writer.packets, 'bytes': writer.bytes, 'avg_packet_size': writer.bytes / float(writer.packets), 'file_size': os.path.getsize(path), 'ipv4_packets': n_ipv4, 'ipv6_packets': n_ipv6, 'vlan_packets': n_vlan, 'flows': len(flows), 'distinct_flow_keys': len(flow_seen), 'flow_head_counts': {str(k): v for k, v in sorted(head_counts.items())}, 'addr_sets': sets, 'addr_set_sizes': size_sets, 'addr_set_sizes_random': size_sets_random, } with open(path + '.manifest.json', 'w') as fh: json.dump(manifest, fh) slim = dict(manifest) slim['addr_sets'] = {k: {kk: vv for kk, vv in v.items() if kk != 'ips'} for k, v in sets.items()} slim['addr_set_sizes'] = {k: {kk: vv for kk, vv in v.items() if kk != 'ips'} for k, v in size_sets.items()} slim['addr_set_sizes_random'] = {k: {kk: vv for kk, vv in v.items() if kk != 'ips'} for k, v in size_sets_random.items()} return slim def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) ap.add_argument('--packets', type=int, default=400000) ap.add_argument('--out', default=os.path.join(os.path.dirname(os.path.abspath(__file__)), 'data', 'bench.pcap')) args = ap.parse_args() outdir = os.path.dirname(os.path.abspath(args.out)) if not os.path.isdir(outdir): os.makedirs(outdir) slim = generate(args.out, args.packets) json.dump(slim, sys.stdout, indent=2, sort_keys=True) sys.stdout.write('\n') if __name__ == '__main__': main() pcapy-ng-2.1.0/benchmarks/probe_analysis.py000066400000000000000000000113271524067405300207430ustar00rootroot00000000000000#!/usr/bin/env python3 """Probe-behaviour analysis for the C IPv4 address set -- no timing involved. The set in pcapobj.cc is an open-addressing table with linear probing: `slots[]` sized to the next power of two >= 2n+1, an index derived from the address, then probe forward until the key or an empty slot is found. Lookup cost is therefore entirely determined by how well the index spreads the keys, which can be computed exactly rather than timed. This script reimplements that table in Python for the old indexing (multiply, then mask the LOW bits -- which discards the mixing) and the current one (multiply, then take the HIGH bits, i.e. Fibonacci hashing), and reports probe counts for member and non-member lookups over sequential (CIDR-like) and random address sets. It is a static analysis: same numbers on any machine, no benchmark noise. python3 probe_analysis.py """ from __future__ import division, print_function import random import struct import sys MASK32 = 0xFFFFFFFF def to_word(ip): """The uint32 the C code actually hashes: memcpy of the 4 network-order bytes on a little-endian host, i.e. the byte-swapped dotted-quad value.""" return struct.unpack('> (32 - bits) def build_table(ips, index): n = len(ips) cap, bits = 16, 4 while cap < n * 2 + 1 and bits < 31: cap <<= 1 bits += 1 slots = [0] * cap mask = cap - 1 worst_insert = 0 for ip in ips: v = to_word(ip) if not v: continue j = index(v, mask, bits) p = 1 while slots[j] and slots[j] != v: j = (j + 1) & mask p += 1 slots[j] = v worst_insert = max(worst_insert, p) return slots, mask, bits, worst_insert def probe_stats(slots, mask, bits, index, keys): """Probes per lookup, counting the initial slot inspection as probe 1.""" counts = [] for ip in keys: v = to_word(ip) j = index(v, mask, bits) p = 1 while slots[j]: if slots[j] == v: break j = (j + 1) & mask p += 1 counts.append(p) counts.sort() n = len(counts) return { 'mean': sum(counts) / n, 'median': counts[n // 2], 'p99': counts[min(n - 1, int(n * 0.99))], 'max': counts[-1], } def sequential_set(size, base='198.18.0.0'): a, b, c, d = (int(x) for x in base.split('.')) start = (a << 24) | (b << 16) | (c << 8) | d return [start + i for i in range(size)] def random_set(size, seed=1234): rnd = random.Random(seed) out, seen = [], set() while len(out) < size: v = rnd.randrange(1, MASK32) if v not in seen: seen.add(v) out.append(v) return out def main(): sizes = [int(x) for x in sys.argv[1:]] or [10, 1000, 100000, 1000000] rnd = random.Random(99) print('%-12s %8s %-6s | %-28s | %-28s' % ('distribution', 'entries', 'hash', 'member lookup probes', 'non-member lookup probes')) print('%-12s %8s %-6s | %-28s | %-28s' % ('', '', '', 'mean / median / p99 / max', 'mean / median / p99 / max')) print('-' * 100) for dist, gen in (('sequential', sequential_set), ('random', random_set)): for size in sizes: ips = gen(size) # non-members: addresses that are not in the set, drawn the same way traffic # would present them (random source/destination addresses) members = set(ips) nonmembers = [] while len(nonmembers) < min(20000, max(1000, size)): v = rnd.randrange(1, MASK32) if v not in members: nonmembers.append(v) probe_members = ips if size <= 20000 else random.Random(7).sample(ips, 20000) for label, fn in (('old', index_old), ('new', index_new)): slots, mask, bits, _ = build_table(ips, fn) m = probe_stats(slots, mask, bits, fn, probe_members) nm = probe_stats(slots, mask, bits, fn, nonmembers) print('%-12s %8d %-6s | %8.2f %6d %6d %6d | %8.2f %6d %6d %6d' % (dist, size, label, m['mean'], m['median'], m['p99'], m['max'], nm['mean'], nm['median'], nm['p99'], nm['max'])) print() if __name__ == '__main__': main() pcapy-ng-2.1.0/benchmarks/pyequiv.py000066400000000000000000000153441524067405300174360ustar00rootroot00000000000000#!/usr/bin/env python3 """Python equivalents of the pcapy-ng in-C classifier paths. These exist so the benchmark compares like with like: whenever loop_filtered() / loop_to_buffer() makes a decision in C, the paired benchmark makes the *same* logical decision in Python on top of the classic API, and both must arrive at the same admitted count (the harness asserts this). The implementations are meant to be what a competent Python author would write: struct parsing with precomputed offsets, plain sets/dicts, no per-packet allocations beyond what is needed. They are not deliberately slow, and they are not micro-tuned beyond readability. Differences from the C classifier that cannot be avoided, and are documented in the report: * the C flow table is a fixed ~12 MB hash (~1M entries) with eviction on collision; the Python dict grows without bound. On this capture the key count stays well under 1M, so both produce identical results, but memory behaviour differs. """ import struct ETH_P_IP = 0x0800 ETH_P_VLAN = 0x8100 ETH_P_QINQ = 0x88A8 _u16 = struct.Struct('!H') _ipv4_addrs = struct.Struct('!II') _ports = struct.Struct('!HH') def parse_ipv4(buf, l2_offset=14): """Return (proto, src, dst, sport, dport) for IPv4 frames, else None. Mirrors what the C classifier does: skip VLAN tags, require IPv4, read the addresses and (for TCP/UDP) the ports out of the L4 header. """ n = len(buf) if n < l2_offset: return None ethertype = _u16.unpack_from(buf, 12)[0] off = l2_offset while ethertype in (ETH_P_VLAN, ETH_P_QINQ): if n < off + 4: return None ethertype = _u16.unpack_from(buf, off + 2)[0] off += 4 if ethertype != ETH_P_IP: return None if n < off + 20: return None vhl = buf[off] if (vhl >> 4) != 4: return None ihl = (vhl & 0x0F) * 4 if ihl < 20 or n < off + ihl: return None proto = buf[off + 9] src, dst = _ipv4_addrs.unpack_from(buf, off + 12) sport = dport = 0 if proto in (6, 17): l4 = off + ihl if n >= l4 + 4: sport, dport = _ports.unpack_from(buf, l4) return proto, src, dst, sport, dport # --- paired equivalents ------------------------------------------------------------------ def classify_all(reader): """Equivalent of loop_filtered(mask=7): classify every packet, admit everything.""" admitted = 0 processed = 0 while True: hdr, data = reader.next() if hdr is None: break processed += 1 parse_ipv4(data) # same work the C classifier does before admitting admitted += 1 return {'processed': processed, 'admitted': admitted, 'delivered': processed} def classify_none(reader): """Equivalent of loop_filtered(mask=0): classify every packet, admit nothing.""" processed = 0 while True: hdr, data = reader.next() if hdr is None: break processed += 1 parse_ipv4(data) return {'processed': processed, 'admitted': 0, 'delivered': processed} def addr_set_match(reader, addr_set): """Equivalent of loop_filtered(admit=SET_MATCH, addr_set=...).""" admitted = 0 processed = 0 contains = addr_set.__contains__ while True: hdr, data = reader.next() if hdr is None: break processed += 1 r = parse_ipv4(data) if r is not None and (contains(r[1]) or contains(r[2])): admitted += 1 return {'processed': processed, 'admitted': admitted, 'delivered': processed} def flow_heads(reader, cutoff): """Equivalent of loop_filtered(admit=FLOW_HEAD, flow_cutoff=N).""" admitted = 0 processed = 0 seen = {} get = seen.get while True: hdr, data = reader.next() if hdr is None: break processed += 1 r = parse_ipv4(data) if r is None or r[0] not in (6, 17): # C tracks flows for TCP/UDP only continue key = (r[1], r[2], r[3], r[4], r[0]) n = get(key, 0) + 1 seen[key] = n if n <= cutoff: admitted += 1 return {'processed': processed, 'admitted': admitted, 'delivered': processed} # --- loop_to_buffer equivalents ---------------------------------------------------------- # # loop_to_buffer writes [u32 caplen LE][u8 class][caplen bytes] records back to back with the # GIL released. The Python equivalent below produces the same bytes in a preallocated # bytearray, so the comparison covers the same copy work. _rec = struct.Struct(' size: overflow += 1 continue _rec.pack_into(buf, pos, n, 0) buf[pos + 5:pos + 5 + n] = data pos += 5 + n written += 1 return {'processed': processed, 'admitted': written, 'delivered': processed, 'overflow': overflow, 'bytes_used': pos} def buffer_addr_set(reader, buf, addr_set): written = 0 processed = 0 pos = 0 size = len(buf) overflow = 0 contains = addr_set.__contains__ while True: hdr, data = reader.next() if hdr is None: break processed += 1 r = parse_ipv4(data) if r is None or not (contains(r[1]) or contains(r[2])): continue n = len(data) if pos + 5 + n > size: overflow += 1 continue _rec.pack_into(buf, pos, n, 2) buf[pos + 5:pos + 5 + n] = data pos += 5 + n written += 1 return {'processed': processed, 'admitted': written, 'delivered': processed, 'overflow': overflow, 'bytes_used': pos} def buffer_flow_heads(reader, buf, cutoff): written = 0 processed = 0 pos = 0 size = len(buf) overflow = 0 seen = {} get = seen.get while True: hdr, data = reader.next() if hdr is None: break processed += 1 r = parse_ipv4(data) if r is None or r[0] not in (6, 17): # C tracks flows for TCP/UDP only continue key = (r[1], r[2], r[3], r[4], r[0]) n = get(key, 0) + 1 seen[key] = n if n > cutoff: continue ln = len(data) if pos + 5 + ln > size: overflow += 1 continue _rec.pack_into(buf, pos, ln, 1) buf[pos + 5:pos + 5 + ln] = data pos += 5 + ln written += 1 return {'processed': processed, 'admitted': written, 'delivered': processed, 'overflow': overflow, 'bytes_used': pos} pcapy-ng-2.1.0/benchmarks/results/000077500000000000000000000000001524067405300170545ustar00rootroot00000000000000pcapy-ng-2.1.0/benchmarks/results/SUMMARY.md000066400000000000000000000231131524067405300205330ustar00rootroot00000000000000# pcapy-ng benchmark results Offline capture, deterministic input, 5 repetitions per scenario, medians reported. ## Environment | item | value | |---|---| | CPU | AMD Ryzen 7 PRO 4750U with Radeon Graphics (16 logical CPUs) | | CPU freq policy | {'scaling_governor': 'schedutil', 'scaling_driver': 'acpi-cpufreq', 'scaling_max_freq_khz': 1700000} | | OS / kernel | Linux-6.8.0-137-generic-x86_64-with-glibc2.39 | | Python | 3.12.3 (CPython) | | libpcap | libpcap version 1.10.4 (with TPACKET_V3) | | libpcap path | `/usr/lib/x86_64-linux-gnu/libpcap.so.1.10.4` | | pcapy-ng | 2.0.2 (git 639a6a7-dirty) | | pcapy module | `/tmp/claude-1000/-home-stamparm-Private-Work-pcapy-ng/d862c1ef-6d01-498b-a4a8-be1665f1e781/scratchpad/v/lib/python3.12/site-packages/pcapy.cpython-312-x86_64-linux-gnu.so` | | load average at start | 2.88 2.26 1.95 | | capture | 400000 packets, 175.2 MiB, avg 443 B/packet | | capture mix | 388066 IPv4, 11934 IPv6, 7860 VLAN-tagged, 23620 distinct flow keys | ## Raw API overhead (NOT equal work) These three APIs do different amounts of work per packet, so this table is about per-call overhead only, not about who is "faster" at the same job. `next()` and `loop()` build a header object and a bytes object per packet; `next_batch(..._meta)` builds two objects per *batch* and only parses the metadata; `next_batch(..._slice)` adds one bytes object per packet so it delivers the same thing `next()` does. | scenario | wall median (s) | spread | processed/s | delivered to Python/s | CPU (s) | peak RSS (MiB) | admitted | dropped | |---|---:|---:|---:|---:|---:|---:|---:|---:| | `raw_next` | 0.2344 | ±3.7% | 1,706,360 | 1,706,360 | 0.234 | 130.9 | 400000 | 0 | | `raw_loop` | 0.1430 | ±29.5% | 2,796,969 | 2,796,969 | 0.143 | 130.9 | 400000 | 0 | | `raw_next_batch_32_meta` | 0.1492 | ±25.4% | 2,680,086 | 0 | 0.149 | 130.9 | 400000 | 0 | | `raw_next_batch_32_slice` | 0.1970 | ±3.6% | 2,030,878 | 2,030,878 | 0.197 | 130.9 | 400000 | 0 | | `raw_next_batch_128_meta` | 0.1409 | ±7.7% | 2,839,053 | 0 | 0.141 | 130.9 | 400000 | 0 | | `raw_next_batch_128_slice` | 0.1882 | ±2.4% | 2,124,959 | 2,124,959 | 0.188 | 130.9 | 400000 | 0 | | `raw_next_batch_512_meta` | 0.1405 | ±2.9% | 2,847,134 | 0 | 0.140 | 130.9 | 400000 | 0 | | `raw_next_batch_512_slice` | 0.1909 | ±5.0% | 2,095,635 | 2,095,635 | 0.191 | 130.9 | 400000 | 0 | | `raw_next_batch_1024_meta` | 0.1425 | ±8.1% | 2,807,836 | 0 | 0.142 | 130.9 | 400000 | 0 | | `raw_next_batch_1024_slice` | 0.1942 | ±7.5% | 2,059,773 | 2,059,773 | 0.194 | 130.9 | 400000 | 0 | ## Classification (equal work: C classifier vs Python equivalent) Each `cls_*` row has a `py_*` counterpart that reaches the same admitted set in Python on top of the classic API. The harness asserts the admitted counts match before any comparison is drawn. | scenario | wall median (s) | spread | processed/s | delivered to Python/s | CPU (s) | peak RSS (MiB) | admitted | dropped | |---|---:|---:|---:|---:|---:|---:|---:|---:| | `cls_admit_all` | 0.1525 | ±11.5% | 2,623,516 | 2,623,516 | 0.152 | 130.9 | 400000 | 0 | | `py_admit_all` | 0.5630 | ±4.8% | 710,438 | 710,438 | 0.563 | 130.9 | 400000 | 0 | | `cls_addrset_50pct` | 0.1146 | ±6.3% | 3,489,847 | 1,745,107 | 0.115 | 130.9 | 200021 | 199979 | | `py_addrset_50pct` | 0.6002 | ±3.7% | 666,400 | 666,400 | 0.600 | 130.9 | 200021 | 199979 | | `cls_addrset_10pct` | 0.0691 | ±9.6% | 5,786,240 | 582,674 | 0.069 | 130.9 | 40280 | 359720 | | `py_addrset_10pct` | 0.6077 | ±1.2% | 658,234 | 658,234 | 0.608 | 130.9 | 40280 | 359720 | | `cls_addrset_1pct` | 0.0549 | ±6.1% | 7,283,987 | 91,887 | 0.055 | 130.9 | 5046 | 394954 | | `py_addrset_1pct` | 0.5929 | ±6.0% | 674,658 | 674,658 | 0.593 | 130.9 | 5046 | 394954 | | `cls_addrset_0pct` | 0.0584 | ±10.7% | 6,844,751 | 0 | 0.058 | 130.9 | 0 | 400000 | | `py_addrset_0pct` | 0.6064 | ±4.0% | 659,623 | 659,623 | 0.606 | 130.9 | 0 | 400000 | | `cls_admit_none` | 0.0495 | ±7.8% | 8,085,182 | 0 | 0.049 | 130.9 | 0 | 400000 | | `py_admit_none` | 0.5659 | ±6.5% | 706,890 | 706,890 | 0.566 | 130.9 | 0 | 400000 | | `cls_flowheads_1` | 0.0879 | ±9.5% | 4,548,354 | 289,696 | 0.088 | 130.9 | 25477 | 374523 | | `py_flowheads_1` | 0.8383 | ±6.5% | 477,131 | 477,131 | 0.838 | 130.9 | 23620 | 376380 | | `cls_flowheads_3` | 0.0945 | ±20.9% | 4,234,171 | 683,109 | 0.094 | 130.9 | 64533 | 335467 | | `py_flowheads_3` | 0.7797 | ±5.1% | 513,025 | 513,025 | 0.779 | 130.9 | 61825 | 338175 | | `cls_setsize_seq_10` | 0.0570 | ±5.4% | 7,016,063 | 193,239 | 0.057 | 130.9 | 11017 | 388983 | | `py_setsize_seq_10` | 0.5992 | ±5.8% | 667,579 | 667,579 | 0.599 | 130.9 | 11017 | 388983 | | `cls_setsize_seq_1000` | 0.0588 | ±4.4% | 6,804,711 | 187,418 | 0.059 | 130.9 | 11017 | 388983 | | `py_setsize_seq_1000` | 0.6074 | ±3.8% | 658,523 | 658,523 | 0.607 | 130.9 | 11017 | 388983 | | `cls_setsize_seq_100000` | 0.0615 | ±21.2% | 6,508,063 | 179,248 | 0.061 | 130.9 | 11017 | 388983 | | `py_setsize_seq_100000` | 0.6256 | ±9.3% | 639,410 | 639,410 | 0.626 | 130.9 | 11017 | 388983 | | `cls_setsize_seq_1000000` | 0.0822 | ±12.5% | 4,867,613 | 134,066 | 0.082 | 238.9 | 11017 | 388983 | | `py_setsize_seq_1000000` | 0.6512 | ±13.1% | 614,273 | 614,273 | 0.651 | 239.0 | 11017 | 388983 | | `cls_setsize_rand_10` | 0.0596 | ±9.3% | 6,715,504 | 184,961 | 0.060 | 130.9 | 11017 | 388983 | | `py_setsize_rand_10` | 0.5937 | ±6.3% | 673,760 | 673,760 | 0.594 | 130.9 | 11017 | 388983 | | `cls_setsize_rand_1000` | 0.0653 | ±6.8% | 6,124,087 | 168,672 | 0.065 | 130.9 | 11017 | 388983 | | `py_setsize_rand_1000` | 0.6141 | ±5.9% | 651,382 | 651,382 | 0.614 | 130.9 | 11017 | 388983 | | `cls_setsize_rand_100000` | 0.0638 | ±8.4% | 6,274,316 | 172,810 | 0.064 | 130.9 | 11017 | 388983 | | `py_setsize_rand_100000` | 0.6233 | ±3.0% | 641,701 | 641,701 | 0.623 | 130.9 | 11017 | 388983 | | `cls_setsize_rand_1000000` | 0.0860 | ±17.3% | 4,650,825 | 128,095 | 0.086 | 238.9 | 11017 | 388983 | | `py_setsize_rand_1000000` | 0.6545 | ±24.3% | 611,153 | 611,153 | 0.654 | 238.9 | 11017 | 388983 | ## loop_to_buffer (equal work: C vs Python copying into a bytearray) `loop_to_buffer` writes `[u32 caplen][u8 class][packet]` records with the GIL released; the Python equivalent produces the same records in a preallocated `bytearray` from the classic API. | scenario | wall median (s) | spread | processed/s | delivered to Python/s | CPU (s) | peak RSS (MiB) | admitted | dropped | |---|---:|---:|---:|---:|---:|---:|---:|---:| | `buf_admit_all` | 0.0646 | ±3.9% | 6,189,041 | 0 | 0.065 | 280.0 | 400000 | 0 | | `py_buf_admit_all` | 0.8310 | ±5.7% | 481,356 | 481,356 | 0.831 | 280.1 | 400000 | 0 | | `buf_addrset_10pct` | 0.0595 | ±12.8% | 6,721,820 | 0 | 0.059 | 280.0 | 40280 | 359720 | | `py_buf_addrset_10pct` | 0.6485 | ±5.1% | 616,852 | 616,852 | 0.648 | 280.2 | 40280 | 359720 | | `buf_flowheads_3` | 0.0718 | ±28.9% | 5,567,552 | 0 | 0.072 | 288.1 | 64533 | 335467 | | `py_buf_flowheads_3` | 0.8682 | ±4.8% | 460,741 | 460,741 | 0.868 | 284.5 | 61825 | 338175 | ## Paired speedups Ratio of Python-equivalent wall time to C wall time, for pairs that admit the same packets. Values below 1.0 mean the C path is *slower* and are reported as such. | C scenario | Python equivalent | admitted (C / Python) | C wall (s) | Python wall (s) | speedup | |---|---|---:|---:|---:|---:| | `cls_admit_all` | `py_admit_all` | 400,000 | 0.1525 | 0.5630 | **3.69x** | | `cls_addrset_50pct` | `py_addrset_50pct` | 200,021 | 0.1146 | 0.6002 | **5.24x** | | `cls_addrset_10pct` | `py_addrset_10pct` | 40,280 | 0.0691 | 0.6077 | **8.79x** | | `cls_addrset_1pct` | `py_addrset_1pct` | 5,046 | 0.0549 | 0.5929 | **10.80x** | | `cls_addrset_0pct` | `py_addrset_0pct` | 0 | 0.0584 | 0.6064 | **10.38x** | | `cls_admit_none` | `py_admit_none` | 0 | 0.0495 | 0.5659 | **11.44x** | | `cls_flowheads_1` | `py_flowheads_1` | 25,477 / 23,620 (+7.86%) | 0.0879 | 0.8383 | **9.53x** | | `cls_flowheads_3` | `py_flowheads_3` | 64,533 / 61,825 (+4.38%) | 0.0945 | 0.7797 | **8.25x** | | `cls_setsize_seq_10` | `py_setsize_seq_10` | 11,017 | 0.0570 | 0.5992 | **10.51x** | | `cls_setsize_seq_1000` | `py_setsize_seq_1000` | 11,017 | 0.0588 | 0.6074 | **10.33x** | | `cls_setsize_seq_100000` | `py_setsize_seq_100000` | 11,017 | 0.0615 | 0.6256 | **10.18x** | | `cls_setsize_seq_1000000` | `py_setsize_seq_1000000` | 11,017 | 0.0822 | 0.6512 | **7.92x** | | `cls_setsize_rand_10` | `py_setsize_rand_10` | 11,017 | 0.0596 | 0.5937 | **9.97x** | | `cls_setsize_rand_1000` | `py_setsize_rand_1000` | 11,017 | 0.0653 | 0.6141 | **9.40x** | | `cls_setsize_rand_100000` | `py_setsize_rand_100000` | 11,017 | 0.0638 | 0.6233 | **9.78x** | | `cls_setsize_rand_1000000` | `py_setsize_rand_1000000` | 11,017 | 0.0860 | 0.6545 | **7.61x** | | `buf_admit_all` | `py_buf_admit_all` | 400,000 | 0.0646 | 0.8310 | **12.86x** | | `buf_addrset_10pct` | `py_buf_addrset_10pct` | 40,280 | 0.0595 | 0.6485 | **10.90x** | | `buf_flowheads_3` | `py_buf_flowheads_3` | 64,533 / 61,825 (+4.38%) | 0.0718 | 0.8682 | **12.08x** | Where the two admitted counts differ, the reason is the C flow table: it is direct-mapped with eviction, so colliding flows restart their counters and a few percent extra packets are reported as flow heads. The Python equivalent uses an exact dict and is the reference count. ## Where the newer API does NOT help None: every paired scenario was at least 1.2x faster in C. (This section exists so that regressions and bad cases show up on their own, not only if someone reads the full table.) ## Correctness cross-check All 48 scenarios admitted the number of packets predicted by the capture manifest (exact match, except the flow-head scenarios where the C table is allowed to admit more; their actual deltas are in the paired table above). pcapy-ng-2.1.0/benchmarks/results/results.csv000066400000000000000000000221561524067405300213000ustar00rootroot00000000000000scenario,group,reps,wall_median_s,wall_min_s,wall_max_s,wall_stdev_s,wall_spread_pct,cpu_median_s,setup_median_s,pkts_processed_per_s,pkts_delivered_per_s,processed,admitted,dropped,delivered,overflow,peak_rss_mib,expected_admitted,expect_mode,admitted_delta_pct,admitted_ok,pair,note raw_next,raw,5,0.234417,0.228871,0.246089,0.007109,7.35,0.234391,5.3e-05,1706360,1706360,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,one Python call + 2 objects per packet raw_loop,raw,5,0.143012,0.133312,0.217811,0.03532,59.08,0.142987,4.5e-05,2796969,2796969,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,one callback + 2 objects per packet raw_next_batch_32_meta,raw,5,0.149249,0.144879,0.220623,0.032648,50.75,0.149237,5.5e-05,2680086,0,400000,400000,0,0,0,130.9,400000,exact,0.0,True,,"metadata parsed, packet bytes not sliced out" raw_next_batch_32_slice,raw,5,0.196959,0.191706,0.205797,0.006396,7.15,0.19693,5.6e-05,2030878,2030878,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,"one bytes object per packet, like next()" raw_next_batch_128_meta,raw,5,0.140892,0.134778,0.156363,0.008051,15.32,0.140892,4.5e-05,2839053,0,400000,400000,0,0,0,130.9,400000,exact,0.0,True,,"metadata parsed, packet bytes not sliced out" raw_next_batch_128_slice,raw,5,0.188239,0.183372,0.192286,0.004051,4.74,0.188201,5.1e-05,2124959,2124959,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,"one bytes object per packet, like next()" raw_next_batch_512_meta,raw,5,0.140492,0.136585,0.144643,0.002856,5.74,0.140439,5.5e-05,2847134,0,400000,400000,0,0,0,130.9,400000,exact,0.0,True,,"metadata parsed, packet bytes not sliced out" raw_next_batch_512_slice,raw,5,0.190873,0.173698,0.192626,0.00875,9.92,0.190855,5.9e-05,2095635,2095635,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,"one bytes object per packet, like next()" raw_next_batch_1024_meta,raw,5,0.142458,0.138157,0.161361,0.009501,16.29,0.142377,4.6e-05,2807836,0,400000,400000,0,0,0,130.9,400000,exact,0.0,True,,"metadata parsed, packet bytes not sliced out" raw_next_batch_1024_slice,raw,5,0.194196,0.187767,0.217089,0.012283,15.1,0.194178,4.6e-05,2059773,2059773,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,,"one bytes object per packet, like next()" cls_admit_all,classify,5,0.152467,0.139351,0.174407,0.013275,22.99,0.152446,4.8e-05,2623516,2623516,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,py_admit_all, py_admit_all,classify,5,0.563033,0.557265,0.61106,0.022689,9.55,0.562985,5.1e-05,710438,710438,400000,400000,0,400000,0,130.9,400000,exact,0.0,True,, cls_addrset_50pct,classify,5,0.114618,0.109596,0.123983,0.005658,12.55,0.114602,0.000134,3489847,1745107,400000,200021,199979,200021,0,130.9,200021,exact,0.0,True,py_addrset_50pct,50.01% of packets admitted py_addrset_50pct,classify,5,0.60024,0.586327,0.630193,0.018538,7.31,0.600157,0.000132,666400,666400,400000,200021,199979,400000,0,130.9,200021,exact,0.0,True,, cls_addrset_10pct,classify,5,0.06913,0.062756,0.076041,0.005915,19.22,0.069106,7.8e-05,5786240,582674,400000,40280,359720,40280,0,130.9,40280,exact,0.0,True,py_addrset_10pct,10.07% of packets admitted py_addrset_10pct,classify,5,0.607686,0.601458,0.61629,0.006225,2.44,0.607631,8.7e-05,658234,658234,400000,40280,359720,400000,0,130.9,40280,exact,0.0,True,, cls_addrset_1pct,classify,5,0.054915,0.049615,0.056365,0.002642,12.29,0.054891,6.1e-05,7283987,91887,400000,5046,394954,5046,0,130.9,5046,exact,0.0,True,py_addrset_1pct,1.26% of packets admitted py_addrset_1pct,classify,5,0.592892,0.590223,0.660835,0.0307,11.91,0.592855,7.8e-05,674658,674658,400000,5046,394954,400000,0,130.9,5046,exact,0.0,True,, cls_addrset_0pct,classify,5,0.058439,0.056501,0.068999,0.005215,21.39,0.058413,0.000195,6844751,0,400000,0,400000,0,0,130.9,0,exact,0.0,True,py_addrset_0pct,0.00% of packets admitted py_addrset_0pct,classify,5,0.606406,0.598403,0.646391,0.022317,7.91,0.606344,0.000242,659623,659623,400000,0,400000,400000,0,130.9,0,exact,0.0,True,, cls_admit_none,classify,5,0.049473,0.048292,0.056014,0.00385,15.61,0.049454,5e-05,8085182,0,400000,0,400000,0,0,130.9,0,exact,0.0,True,py_admit_none,"classify everything, deliver nothing" py_admit_none,classify,5,0.565859,0.543316,0.617344,0.032459,13.08,0.565804,4.7e-05,706890,706890,400000,0,400000,400000,0,130.9,0,exact,0.0,True,, cls_flowheads_1,classify,5,0.087944,0.074143,0.090835,0.007531,18.98,0.087905,5.6e-05,4548354,289696,400000,25477,374523,25477,0,130.9,23620,min,7.862,True,py_flowheads_1,C table is direct-mapped with eviction; admits >= exact count py_flowheads_1,classify,5,0.838343,0.79212,0.900733,0.044034,12.96,0.838304,4.8e-05,477131,477131,400000,23620,376380,400000,0,130.9,23620,exact,0.0,True,, cls_flowheads_3,classify,5,0.094469,0.084386,0.123834,0.014987,41.76,0.094448,5.3e-05,4234171,683109,400000,64533,335467,64533,0,130.9,61825,min,4.38,True,py_flowheads_3,C table is direct-mapped with eviction; admits >= exact count py_flowheads_3,classify,5,0.779688,0.754363,0.834075,0.031593,10.22,0.77899,4.9e-05,513025,513025,400000,61825,338175,400000,0,130.9,61825,exact,0.0,True,, cls_setsize_seq_10,classify,5,0.057012,0.055948,0.062105,0.002539,10.8,0.056986,6.8e-05,7016063,193239,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_seq_10,"10 addresses, consecutive padding, 2.75% admitted" py_setsize_seq_10,classify,5,0.599179,0.581128,0.65014,0.027929,11.52,0.599151,5.9e-05,667579,667579,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_seq_1000,classify,5,0.058783,0.0582,0.063411,0.00218,8.87,0.058758,0.00019,6804711,187418,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_seq_1000,"1000 addresses, consecutive padding, 2.75% admitted" py_setsize_seq_1000,classify,5,0.60742,0.600786,0.64709,0.019074,7.62,0.607368,0.000176,658523,658523,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_seq_100000,classify,5,0.061462,0.059566,0.085658,0.01114,42.45,0.061467,0.015902,6508063,179248,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_seq_100000,"100000 addresses, consecutive padding, 2.75% admitted" py_setsize_seq_100000,classify,5,0.625576,0.609206,0.72562,0.047531,18.61,0.625533,0.017381,639410,639410,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_seq_1000000,classify,5,0.082176,0.074516,0.095131,0.007465,25.09,0.082168,0.186716,4867613,134066,400000,11017,388983,11017,0,238.9,11017,exact,0.0,True,py_setsize_seq_1000000,"1000000 addresses, consecutive padding, 2.75% admitted" py_setsize_seq_1000000,classify,5,0.651176,0.621575,0.792211,0.06697,26.2,0.651151,0.19061,614273,614273,400000,11017,388983,400000,0,239.0,11017,exact,0.0,True,, cls_setsize_rand_10,classify,5,0.059564,0.053091,0.064177,0.004661,18.61,0.059536,6.1e-05,6715504,184961,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_rand_10,"10 addresses, random padding, 2.75% admitted" py_setsize_rand_10,classify,5,0.593683,0.592251,0.666971,0.032392,12.59,0.593664,7.7e-05,673760,673760,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_rand_1000,classify,5,0.065316,0.059931,0.068828,0.003738,13.62,0.065292,0.000218,6124087,168672,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_rand_1000,"1000 addresses, random padding, 2.75% admitted" py_setsize_rand_1000,classify,5,0.614079,0.593518,0.666542,0.029363,11.89,0.613796,0.00019,651382,651382,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_rand_100000,classify,5,0.063752,0.058453,0.069156,0.004078,16.79,0.063757,0.021056,6274316,172810,400000,11017,388983,11017,0,130.9,11017,exact,0.0,True,py_setsize_rand_100000,"100000 addresses, random padding, 2.75% admitted" py_setsize_rand_100000,classify,5,0.623343,0.594286,0.631896,0.015805,6.03,0.623305,0.022057,641701,641701,400000,11017,388983,400000,0,130.9,11017,exact,0.0,True,, cls_setsize_rand_1000000,classify,5,0.086006,0.076353,0.106113,0.011742,34.6,0.086011,0.26053,4650825,128095,400000,11017,388983,11017,0,238.9,11017,exact,0.0,True,py_setsize_rand_1000000,"1000000 addresses, random padding, 2.75% admitted" py_setsize_rand_1000000,classify,5,0.6545,0.609506,0.927965,0.131676,48.66,0.654497,0.269204,611153,611153,400000,11017,388983,400000,0,238.9,11017,exact,0.0,True,, buf_admit_all,buffer,5,0.06463,0.063135,0.068203,0.00236,7.84,0.064636,0.053028,6189041,0,400000,400000,0,0,0,280.0,400000,exact,0.0,True,py_buf_admit_all, py_buf_admit_all,buffer,5,0.830985,0.810635,0.904815,0.038619,11.33,0.830964,0.053328,481356,481356,400000,400000,0,400000,0,280.1,400000,exact,0.0,True,, buf_addrset_10pct,buffer,5,0.059508,0.05343,0.068698,0.005937,25.66,0.059446,0.057064,6721820,0,400000,40280,359720,0,0,280.0,40280,exact,0.0,True,py_buf_addrset_10pct, py_buf_addrset_10pct,buffer,5,0.648453,0.623002,0.688751,0.024802,10.14,0.647812,0.05584,616852,616852,400000,40280,359720,400000,0,280.2,40280,exact,0.0,True,, buf_flowheads_3,buffer,5,0.071845,0.065987,0.107461,0.016736,57.73,0.071826,0.055291,5567552,0,400000,64533,335467,0,0,288.1,61825,min,4.38,True,py_buf_flowheads_3,C table is direct-mapped with eviction; admits >= exact count py_buf_flowheads_3,buffer,5,0.868165,0.836838,0.919977,0.032359,9.58,0.868141,0.055304,460741,460741,400000,61825,338175,400000,0,284.5,61825,exact,0.0,True,, pcapy-ng-2.1.0/benchmarks/results/results.json000066400000000000000000002653441524067405300214660ustar00rootroot00000000000000{ "results": [ { "admitted": 400000, "cpu": { "max": 0.24602896900000004, "mean": 0.23527867800000002, "median": 0.234391464, "min": 0.22883095500000003, "rel_spread_pct": 7.33730388748287, "stdev": 0.007104064576493278 }, "delivered": 400000, "dropped": 0, "name": "raw_next", "overflow": 0, "peak_rss_kib_max": 134048, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.22887059900676832, 0.2344170349970227, 0.22927075999905355, 0.24608944500505459, 0.2379955219948897 ], "setup_s_median": 5.257800512481481e-05, "wall": { "max": 0.24608944500505459, "mean": 0.23532867220055778, "median": 0.2344170349970227, "min": 0.22887059900676832, "rel_spread_pct": 7.34539023518703, "stdev": 0.007108839764403742 } }, { "admitted": 400000, "cpu": { "max": 0.21776857200000002, "mean": 0.1551338226, "median": 0.14298708, "min": 0.133303278, "rel_spread_pct": 59.07197629324274, "stdev": 0.03530900383487821 }, "delivered": 400000, "dropped": 0, "name": "raw_loop", "overflow": 0, "peak_rss_kib_max": 134068, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.13331248599570245, 0.1370156259945361, 0.14463738900667522, 0.14301193699066062, 0.21781103800458368 ], "setup_s_median": 4.512499435804784e-05, "wall": { "max": 0.21781103800458368, "mean": 0.15515769519843162, "median": 0.14301193699066062, "min": 0.13331248599570245, "rel_spread_pct": 59.084964365177015, "stdev": 0.035319914671843027 } }, { "admitted": 400000, "cpu": { "max": 0.22044219099999995, "mean": 0.1622873298, "median": 0.14923723400000002, "min": 0.144878748, "rel_spread_pct": 50.633103398311405, "stdev": 0.03257217542212572 }, "delivered": 0, "dropped": 0, "name": "raw_next_batch_32_meta", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.22062326500599738, 0.14689172100042924, 0.14487880299566314, 0.14924893600982614, 0.150024124988704 ], "setup_s_median": 5.496400990523398e-05, "wall": { "max": 0.22062326500599738, "mean": 0.162333370000124, "median": 0.14924893600982614, "min": 0.14487880299566314, "rel_spread_pct": 50.75042009367989, "stdev": 0.03264788744519453 } }, { "admitted": 400000, "cpu": { "max": 0.20575295700000001, "mean": 0.19843751140000002, "median": 0.19693018400000004, "min": 0.19168105300000005, "rel_spread_pct": 7.145630859716237, "stdev": 0.006383208752420463 }, "delivered": 400000, "dropped": 0, "name": "raw_next_batch_32_slice", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.20449151600769255, 0.191706260011415, 0.19340536500385497, 0.20579718198860064, 0.19695908100402448 ], "setup_s_median": 5.626499478239566e-05, "wall": { "max": 0.20579718198860064, "mean": 0.19847188080311753, "median": 0.19695908100402448, "min": 0.191706260011415, "rel_spread_pct": 7.154238284091975, "stdev": 0.00639585443329564 } }, { "admitted": 400000, "cpu": { "max": 0.15633271800000004, "mean": 0.14274347199999998, "median": 0.14089222099999998, "min": 0.134757102, "rel_spread_pct": 15.313560853015476, "stdev": 0.008046598404124805 }, "delivered": 0, "dropped": 0, "name": "raw_next_batch_128_meta", "overflow": 0, "peak_rss_kib_max": 134072, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.14089204599440563, 0.1347784030076582, 0.14109264899161644, 0.14069014599954244, 0.156363076006528 ], "setup_s_median": 4.510399594437331e-05, "wall": { "max": 0.156363076006528, "mean": 0.14276326399995015, "median": 0.14089204599440563, "min": 0.1347784030076582, "rel_spread_pct": 15.320008199559291, "stdev": 0.008051445852367802 } }, { "admitted": 400000, "cpu": { "max": 0.19226904900000003, "mean": 0.18804275480000002, "median": 0.188200677, "min": 0.183333996, "rel_spread_pct": 4.7476200098897765, "stdev": 0.004056571690281738 }, "delivered": 400000, "dropped": 0, "name": "raw_next_batch_128_slice", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.1882388709927909, 0.18337248200259637, 0.18464048799069133, 0.19181156500417273, 0.19228637100604828 ], "setup_s_median": 5.085500015411526e-05, "wall": { "max": 0.19228637100604828, "mean": 0.1880699553992599, "median": 0.1882388709927909, "min": 0.18337248200259637, "rel_spread_pct": 4.735413550049019, "stdev": 0.0040507356893926755 } }, { "admitted": 400000, "cpu": { "max": 0.14462937799999998, "mean": 0.14044517339999998, "median": 0.14043866699999996, "min": 0.13656646599999997, "rel_spread_pct": 5.741233644719803, "stdev": 0.0028578014946812357 }, "delivered": 0, "dropped": 0, "name": "raw_next_batch_512_meta", "overflow": 0, "peak_rss_kib_max": 134060, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.14464261299872305, 0.13658531800319906, 0.1401217750099022, 0.14050079000298865, 0.14049211599922273 ], "setup_s_median": 5.4553005611523986e-05, "wall": { "max": 0.14464261299872305, "mean": 0.14046852240280713, "median": 0.14049211599922273, "min": 0.13658531800319906, "rel_spread_pct": 5.735051350190046, "stdev": 0.0028558772873824977 } }, { "admitted": 400000, "cpu": { "max": 0.19259215600000001, "mean": 0.1858145322, "median": 0.19085501899999996, "min": 0.17368632199999998, "rel_spread_pct": 9.905861579673752, "stdev": 0.008745539918622735 }, "delivered": 400000, "dropped": 0, "name": "raw_next_batch_512_slice", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.17369843300548382, 0.19259641200187616, 0.1793738529959228, 0.1908728609996615, 0.1926255489961477 ], "setup_s_median": 5.930199404247105e-05, "wall": { "max": 0.1926255489961477, "mean": 0.1858334215998184, "median": 0.1908728609996615, "min": 0.17369843300548382, "rel_spread_pct": 9.916085446373355, "stdev": 0.008750013684904264 } }, { "admitted": 400000, "cpu": { "max": 0.161320317, "mean": 0.14629778599999999, "median": 0.142376923, "min": 0.13813368299999998, "rel_spread_pct": 16.285387766105895, "stdev": 0.009499308344855902 }, "delivered": 0, "dropped": 0, "name": "raw_next_batch_1024_meta", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.14979123600642197, 0.138156640008674, 0.14245841599768028, 0.13989991099515464, 0.16136144699703436 ], "setup_s_median": 4.560600791592151e-05, "wall": { "max": 0.16136144699703436, "mean": 0.14633353000099306, "median": 0.14245841599768028, "min": 0.138156640008674, "rel_spread_pct": 16.288828445725674, "stdev": 0.009500835648211818 } }, { "admitted": 400000, "cpu": { "max": 0.21705531200000003, "mean": 0.198711164, "median": 0.19417844899999998, "min": 0.18774217599999998, "rel_spread_pct": 15.095978029982128, "stdev": 0.012277826477654878 }, "delivered": 400000, "dropped": 0, "name": "raw_next_batch_1024_slice", "overflow": 0, "peak_rss_kib_max": 134076, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.1877667439985089, 0.1941960979893338, 0.18952259099751245, 0.21708934799244162, 0.2051214569946751 ], "setup_s_median": 4.620700201485306e-05, "wall": { "max": 0.21708934799244162, "mean": 0.19873924759449438, "median": 0.1941960979893338, "min": 0.1877667439985089, "rel_spread_pct": 15.099481553714462, "stdev": 0.012282524005349776 } }, { "admitted": 400000, "cpu": { "max": 0.17437572900000003, "mean": 0.15294174040000003, "median": 0.152446399, "min": 0.139333929, "rel_spread_pct": 22.98630878122614, "stdev": 0.013270124546783843 }, "delivered": 400000, "dropped": 0, "name": "cls_admit_all", "overflow": 0, "peak_rss_kib_max": 134064, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.15246713400119916, 0.13935055299953092, 0.14525680300721433, 0.17440703400643542, 0.15334146800159942 ], "setup_s_median": 4.8080997657962143e-05, "wall": { "max": 0.17440703400643542, "mean": 0.15296459840319585, "median": 0.15246713400119916, "min": 0.13935055299953092, "rel_spread_pct": 22.992811687946325, "stdev": 0.013275163730100422 } }, { "admitted": 400000, "cpu": { "max": 0.611013184, "mean": 0.5752842144, "median": 0.5629848719999999, "min": 0.5572386870000001, "rel_spread_pct": 9.551677083074434, "stdev": 0.022687625409518433 }, "delivered": 400000, "dropped": 0, "name": "py_admit_all", "overflow": 0, "peak_rss_kib_max": 134076, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5605949009914184, 0.5630325220117811, 0.5572653980052564, 0.6110597700026119, 0.5847068279981613 ], "setup_s_median": 5.1355993491597474e-05, "wall": { "max": 0.6110597700026119, "mean": 0.5753318838018459, "median": 0.5630325220117811, "min": 0.5572653980052564, "rel_spread_pct": 9.554398705982718, "stdev": 0.02268932735282728 } }, { "admitted": 200021, "cpu": { "max": 0.12393075199999998, "mean": 0.1148544978, "median": 0.11460165800000005, "min": 0.109575156, "rel_spread_pct": 12.526516850218663, "stdev": 0.005648358829410637 }, "delivered": 200021, "dropped": 199979, "name": "cls_addrset_50pct", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.11544753199268598, 0.11077051699976437, 0.10959627300326247, 0.123983326004236, 0.11461818299721926 ], "setup_s_median": 0.00013413200213108212, "wall": { "max": 0.123983326004236, "mean": 0.11488316619943362, "median": 0.11461818299721926, "min": 0.10959627300326247, "rel_spread_pct": 12.552155883785542, "stdev": 0.0056583291797634725 } }, { "admitted": 200021, "cpu": { "max": 0.6301063610000001, "mean": 0.6053427246, "median": 0.6001571480000001, "min": 0.586307462, "rel_spread_pct": 7.297905081353809, "stdev": 0.018514460017062726 }, "delivered": 400000, "dropped": 199979, "name": "py_addrset_50pct", "overflow": 0, "peak_rss_kib_max": 134068, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5915062470012344, 0.6186832490056986, 0.586326820004615, 0.6002395810064627, 0.6301927210006397 ], "setup_s_median": 0.00013210800534579903, "wall": { "max": 0.6301927210006397, "mean": 0.6053897236037301, "median": 0.6002395810064627, "min": 0.586326820004615, "rel_spread_pct": 7.308065376573762, "stdev": 0.018537635248712787 } }, { "admitted": 40280, "cpu": { "max": 0.076034449, "mean": 0.0690315582, "median": 0.06910550899999998, "min": 0.06272561200000001, "rel_spread_pct": 19.258720748298074, "stdev": 0.005922249240359505 }, "delivered": 40280, "dropped": 359720, "name": "cls_addrset_10pct", "overflow": 0, "peak_rss_kib_max": 134064, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.06275608501164243, 0.06912952099810354, 0.06360352999763563, 0.07371865599998273, 0.07604069000808522 ], "setup_s_median": 7.810600800439715e-05, "wall": { "max": 0.07604069000808522, "mean": 0.06904969640308992, "median": 0.06912952099810354, "min": 0.06275608501164243, "rel_spread_pct": 19.216978223828914, "stdev": 0.005915019444831274 } }, { "admitted": 40280, "cpu": { "max": 0.6162594299999999, "mean": 0.6088313949999999, "median": 0.6076310229999999, "min": 0.601408321, "rel_spread_pct": 2.4440998628866875, "stdev": 0.006224743432371844 }, "delivered": 400000, "dropped": 359720, "name": "py_addrset_10pct", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6076864969945746, 0.61409149300016, 0.6048296439985279, 0.6162899979972281, 0.6014577519963495 ], "setup_s_median": 8.696400618646294e-05, "wall": { "max": 0.6162899979972281, "mean": 0.608871076797368, "median": 0.6076864969945746, "min": 0.6014577519963495, "rel_spread_pct": 2.440772680359711, "stdev": 0.006224687343197005 } }, { "admitted": 5046, "cpu": { "max": 0.056341295999999985, "mean": 0.0539217128, "median": 0.054890766000000035, "min": 0.04959891599999999, "rel_spread_pct": 12.283268191229082, "stdev": 0.002637290753863655 }, "delivered": 5046, "dropped": 394954, "name": "cls_addrset_1pct", "overflow": 0, "peak_rss_kib_max": 134076, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.05636483799025882, 0.055394474999047816, 0.0549149759899592, 0.04961453798750881, 0.053418836992932484 ], "setup_s_median": 6.080399907659739e-05, "wall": { "max": 0.05636483799025882, "mean": 0.05394153279194143, "median": 0.0549149759899592, "min": 0.04961453798750881, "rel_spread_pct": 12.292275251079507, "stdev": 0.0026423065556932836 } }, { "admitted": 5046, "cpu": { "max": 0.660817896, "mean": 0.6111086014, "median": 0.592854926, "min": 0.590183302, "rel_spread_pct": 11.914313418389296, "stdev": 0.030703968221882382 }, "delivered": 400000, "dropped": 394954, "name": "py_addrset_1pct", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.590462366992142, 0.5928921140002785, 0.5902232619991992, 0.6212622409948381, 0.660834624999552 ], "setup_s_median": 7.849698886275291e-05, "wall": { "max": 0.660834624999552, "mean": 0.611134921797202, "median": 0.5928921140002785, "min": 0.5902232619991992, "rel_spread_pct": 11.909647865601345, "stdev": 0.030700468915488884 } }, { "admitted": 0, "cpu": { "max": 0.06897648499999998, "mean": 0.059868449600000005, "median": 0.05841291000000004, "min": 0.056479133999999986, "rel_spread_pct": 21.39484405074149, "stdev": 0.005213297986857084 }, "delivered": 0, "dropped": 400000, "name": "cls_addrset_0pct", "overflow": 0, "peak_rss_kib_max": 134072, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.058438934007426724, 0.05650081498606596, 0.056515634001698345, 0.05897438399551902, 0.0689993309933925 ], "setup_s_median": 0.00019485599477775395, "wall": { "max": 0.0689993309933925, "mean": 0.05988581959682051, "median": 0.058438934007426724, "min": 0.05650081498606596, "rel_spread_pct": 21.38731005212752, "stdev": 0.005215282588219965 } }, { "admitted": 0, "cpu": { "max": 0.646332023, "mean": 0.6188812526, "median": 0.606344295, "min": 0.59839281, "rel_spread_pct": 7.906269325087002, "stdev": 0.022302243568715466 }, "delivered": 400000, "dropped": 400000, "name": "py_addrset_0pct", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6037433190067532, 0.6396466579899425, 0.5984026189980796, 0.6064062080113217, 0.6463910979946377 ], "setup_s_median": 0.0002417840005364269, "wall": { "max": 0.6463910979946377, "mean": 0.6189179804001469, "median": 0.6064062080113217, "min": 0.5984026189980796, "rel_spread_pct": 7.913586365471733, "stdev": 0.022316726435309506 } }, { "admitted": 0, "cpu": { "max": 0.056003440000000015, "mean": 0.05152915160000001, "median": 0.049454233000000014, "min": 0.04827104800000001, "rel_spread_pct": 15.635450255592888, "stdev": 0.003852957563790609 }, "delivered": 0, "dropped": 400000, "name": "cls_admit_none", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.0494732200022554, 0.05601364000176545, 0.048292264007613994, 0.055451394990086555, 0.04851304000476375 ], "setup_s_median": 4.9883994506672025e-05, "wall": { "max": 0.05601364000176545, "mean": 0.05154871180129703, "median": 0.0494732200022554, "min": 0.048292264007613994, "rel_spread_pct": 15.607183025077918, "stdev": 0.0038501279720464586 } }, { "admitted": 0, "cpu": { "max": 0.6172856010000001, "mean": 0.5733591888, "median": 0.565803624, "min": 0.543289895, "rel_spread_pct": 13.07798374935825, "stdev": 0.03245518045934529 }, "delivered": 400000, "dropped": 400000, "name": "py_admit_none", "overflow": 0, "peak_rss_kib_max": 134048, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5447431709908415, 0.595763999997871, 0.5433158919913694, 0.6173441999999341, 0.5658588980004424 ], "setup_s_median": 4.740900476463139e-05, "wall": { "max": 0.6173441999999341, "mean": 0.5734052321960916, "median": 0.5658588980004424, "min": 0.5433158919913694, "rel_spread_pct": 13.082467779539407, "stdev": 0.032459115420642747 } }, { "admitted": 25477, "cpu": { "max": 0.09081230000000001, "mean": 0.084260792, "median": 0.08790541499999999, "min": 0.07412397700000001, "rel_spread_pct": 18.98440841215528, "stdev": 0.007546080230708285 }, "delivered": 25477, "dropped": 374523, "name": "cls_flowheads_1", "overflow": 0, "peak_rss_kib_max": 134068, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.07414347000303678, 0.08794388998649083, 0.0784864230081439, 0.09012286800134461, 0.09083482800633647 ], "setup_s_median": 5.64860092708841e-05, "wall": { "max": 0.09083482800633647, "mean": 0.08430629580107052, "median": 0.08794388998649083, "min": 0.07414347000303678, "rel_spread_pct": 18.979553901770394, "stdev": 0.007530608937878584 } }, { "admitted": 23620, "cpu": { "max": 0.900558497, "mean": 0.8341718174, "median": 0.8383039950000001, "min": 0.792082521, "rel_spread_pct": 12.939933084775532, "stdev": 0.04398129808046187 }, "delivered": 400000, "dropped": 376380, "name": "py_flowheads_1", "overflow": 0, "peak_rss_kib_max": 134072, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.7961563559947535, 0.8383427719963947, 0.7921201639983337, 0.9007327970030019, 0.8439008550049039 ], "setup_s_median": 4.831099067814648e-05, "wall": { "max": 0.9007327970030019, "mean": 0.8342505887994776, "median": 0.8383427719963947, "min": 0.7921201639983337, "rel_spread_pct": 12.955635407462577, "stdev": 0.04403379632644348 } }, { "admitted": 64533, "cpu": { "max": 0.123836385, "mean": 0.09813766839999999, "median": 0.09444759599999997, "min": 0.084369164, "rel_spread_pct": 41.78742781340883, "stdev": 0.014995330626743955 }, "delivered": 64533, "dropped": 335467, "name": "cls_flowheads_3", "overflow": 0, "peak_rss_kib_max": 134072, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.09446947899414226, 0.0843862969923066, 0.09501829399960116, 0.12383370599127375, 0.09304874700319488 ], "setup_s_median": 5.298899486660957e-05, "wall": { "max": 0.12383370599127375, "mean": 0.09815130459610373, "median": 0.09446947899414226, "min": 0.0843862969923066, "rel_spread_pct": 41.75677628264802, "stdev": 0.014987235109462585 } }, { "admitted": 61825, "cpu": { "max": 0.83401016, "mean": 0.7879698852, "median": 0.778989542, "min": 0.7543018110000002, "rel_spread_pct": 10.232274594515646, "stdev": 0.03163296181275123 }, "delivered": 400000, "dropped": 338175, "name": "py_flowheads_3", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.7543632560118567, 0.7796880110108759, 0.7680941540020285, 0.8340754880046006, 0.804519472992979 ], "setup_s_median": 4.879100015386939e-05, "wall": { "max": 0.8340754880046006, "mean": 0.7881480764044682, "median": 0.7796880110108759, "min": 0.7543632560118567, "rel_spread_pct": 10.223606220313165, "stdev": 0.031593101943676836 } }, { "admitted": 11017, "cpu": { "max": 0.06208560799999996, "mean": 0.057628828, "median": 0.05698647800000001, "min": 0.05592462100000001, "rel_spread_pct": 10.811313869932356, "stdev": 0.0025396855636325116 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_seq_10", "overflow": 0, "peak_rss_kib_max": 134048, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.05701202400086913, 0.05703306400391739, 0.05615113901149016, 0.06210499600274488, 0.05594797000230756 ], "setup_s_median": 6.764699355699122e-05, "wall": { "max": 0.06210499600274488, "mean": 0.05764983860426583, "median": 0.05701202400086913, "min": 0.05594797000230756, "rel_spread_pct": 10.799521869883911, "stdev": 0.002538607050573261 } }, { "admitted": 11017, "cpu": { "max": 0.650035829, "mean": 0.60490543, "median": 0.599150566, "min": 0.581098186, "rel_spread_pct": 11.505896332575615, "stdev": 0.027900982466106557 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_seq_10", "overflow": 0, "peak_rss_kib_max": 134068, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5811282419890631, 0.5838160649873316, 0.5991793810098898, 0.6501396999956341, 0.6104850260016974 ], "setup_s_median": 5.89400005992502e-05, "wall": { "max": 0.6501396999956341, "mean": 0.6049496827967232, "median": 0.5991793810098898, "min": 0.5811282419890631, "rel_spread_pct": 11.517662355179072, "stdev": 0.027929218147900867 } }, { "admitted": 11017, "cpu": { "max": 0.06339900700000001, "mean": 0.059721481199999996, "median": 0.05875832, "min": 0.05818121499999995, "rel_spread_pct": 8.880090513139338, "stdev": 0.0021830972948678287 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_seq_1000", "overflow": 0, "peak_rss_kib_max": 134060, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.05820018699159846, 0.0582699880033033, 0.0587828029965749, 0.060019687996828, 0.06341137300478294 ], "setup_s_median": 0.0001898869959404692, "wall": { "max": 0.06341137300478294, "mean": 0.05973680779861752, "median": 0.0587828029965749, "min": 0.05820018699159846, "rel_spread_pct": 8.865154003438946, "stdev": 0.0021797556995974974 } }, { "admitted": 11017, "cpu": { "max": 0.64706466, "mean": 0.6149299474, "median": 0.607368328, "min": 0.600719517, "rel_spread_pct": 7.630483985328904, "stdev": 0.019083535030314194 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_seq_1000", "overflow": 0, "peak_rss_kib_max": 134068, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6074196190020302, 0.6023307499999646, 0.6007861499965657, 0.6173424090084154, 0.6470896629907656 ], "setup_s_median": 0.00017648100038059056, "wall": { "max": 0.6470896629907656, "mean": 0.6149937181995483, "median": 0.6074196190020302, "min": 0.6007861499965657, "rel_spread_pct": 7.622986078433734, "stdev": 0.019073795879926757 } }, { "admitted": 11017, "cpu": { "max": 0.08565651400000002, "mean": 0.06578891560000001, "median": 0.061467245000000004, "min": 0.059570818000000025, "rel_spread_pct": 42.438368597779174, "stdev": 0.011136556726865149 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_seq_100000", "overflow": 0, "peak_rss_kib_max": 134060, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.06060851100482978, 0.061462215991923586, 0.059565553994616494, 0.0856584209977882, 0.06163284099602606 ], "setup_s_median": 0.015902066996204667, "wall": { "max": 0.0856584209977882, "mean": 0.06578550859703683, "median": 0.061462215991923586, "min": 0.059565553994616494, "rel_spread_pct": 42.45350835804623, "stdev": 0.011139538744577088 } }, { "admitted": 11017, "cpu": { "max": 0.7255926799999999, "mean": 0.64286296, "median": 0.625532948, "min": 0.609146475, "rel_spread_pct": 18.61551903417882, "stdev": 0.04753688218019544 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_seq_100000", "overflow": 0, "peak_rss_kib_max": 134052, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6157262570050079, 0.7256196589878527, 0.6255762669898104, 0.6383683439926244, 0.6092059140064521 ], "setup_s_median": 0.017381224999553524, "wall": { "max": 0.7256196589878527, "mean": 0.6428992881963496, "median": 0.6255762669898104, "min": 0.6092059140064521, "rel_spread_pct": 18.609041155216467, "stdev": 0.047531287988569496 } }, { "admitted": 11017, "cpu": { "max": 0.09513800500000003, "mean": 0.08311141200000002, "median": 0.08216822400000001, "min": 0.07451988100000001, "rel_spread_pct": 25.092575933003022, "stdev": 0.007468789229264209 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_seq_1000000", "overflow": 0, "peak_rss_kib_max": 244640, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.08129814299172722, 0.09513059401069768, 0.08217578499170486, 0.08247854599903803, 0.07451610699354205 ], "setup_s_median": 0.1867162140115397, "wall": { "max": 0.09513059401069768, "mean": 0.08311983499734196, "median": 0.08217578499170486, "min": 0.07451610699354205, "rel_spread_pct": 25.08584130864893, "stdev": 0.007464976748056321 } }, { "admitted": 11017, "cpu": { "max": 0.7921905960000001, "mean": 0.6775827924000001, "median": 0.651151408, "min": 0.621549148, "rel_spread_pct": 26.20610904061809, "stdev": 0.06697028178999383 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_seq_1000000", "overflow": 0, "peak_rss_kib_max": 244736, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6465147019916913, 0.792211022999254, 0.6215752210118808, 0.6765030950045912, 0.6511760319990572 ], "setup_s_median": 0.19060972200531978, "wall": { "max": 0.792211022999254, "mean": 0.6775960146012949, "median": 0.6511760319990572, "min": 0.6215752210118808, "rel_spread_pct": 26.2042510169693, "stdev": 0.06697049240511693 } }, { "admitted": 11017, "cpu": { "max": 0.06415956700000003, "mean": 0.058831186800000004, "median": 0.059536131999999964, "min": 0.05307043399999997, "rel_spread_pct": 18.625887553460917, "stdev": 0.004659862061823806 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_rand_10", "overflow": 0, "peak_rss_kib_max": 134076, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.05519712399109267, 0.06417675699049141, 0.0530909299995983, 0.059563652001088485, 0.06224114399810787 ], "setup_s_median": 6.051399395801127e-05, "wall": { "max": 0.06417675699049141, "mean": 0.058853921396075745, "median": 0.059563652001088485, "min": 0.0530909299995983, "rel_spread_pct": 18.61173151486837, "stdev": 0.00466084247613968 } }, { "admitted": 11017, "cpu": { "max": 0.6669306719999999, "mean": 0.6093172607999999, "median": 0.593664117, "min": 0.592066092, "rel_spread_pct": 12.61059542866052, "stdev": 0.032406385276706466 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_rand_10", "overflow": 0, "peak_rss_kib_max": 134048, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6010854209889658, 0.666970758989919, 0.592250995003269, 0.5928968209918821, 0.5936826480028685 ], "setup_s_median": 7.687498873565346e-05, "wall": { "max": 0.666970758989919, "mean": 0.6093773287953809, "median": 0.5936826480028685, "min": 0.592250995003269, "rel_spread_pct": 12.585808973532421, "stdev": 0.032392130172474345 } }, { "admitted": 11017, "cpu": { "max": 0.06882852900000003, "mean": 0.06427231380000001, "median": 0.065292458, "min": 0.05992733899999997, "rel_spread_pct": 13.63279967190094, "stdev": 0.0037404319269772573 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_rand_1000", "overflow": 0, "peak_rss_kib_max": 134076, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.059931499999947846, 0.06882784600020386, 0.060981153990724124, 0.06531584999174811, 0.06636654501198791 ], "setup_s_median": 0.0002183410106226802, "wall": { "max": 0.06882784600020386, "mean": 0.06428457899892237, "median": 0.06531584999174811, "min": 0.059931499999947846, "rel_spread_pct": 13.62050099842529, "stdev": 0.0037383582941325913 } }, { "admitted": 11017, "cpu": { "max": 0.6664787130000001, "mean": 0.6168572224000001, "median": 0.613796378, "min": 0.593329859, "rel_spread_pct": 11.917446342441604, "stdev": 0.02938825496175726 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_rand_1000", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5963044579984853, 0.6665417339972919, 0.5935178460058523, 0.6145239460020093, 0.6140789210039657 ], "setup_s_median": 0.00018974600243382156, "wall": { "max": 0.6665417339972919, "mean": 0.6169933810015209, "median": 0.6140789210039657, "min": 0.5935178460058523, "rel_spread_pct": 11.891612868269739, "stdev": 0.02936308800707993 } }, { "admitted": 11017, "cpu": { "max": 0.06916314800000001, "mean": 0.063175, "median": 0.06375656900000004, "min": 0.05845653199999995, "rel_spread_pct": 16.792961365283084, "stdev": 0.004079185709014447 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_rand_100000", "overflow": 0, "peak_rss_kib_max": 134056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.06915593000303488, 0.06403082300676033, 0.06375195999862626, 0.06045794300734997, 0.05845280599896796 ], "setup_s_median": 0.02105598100752104, "wall": { "max": 0.06915593000303488, "mean": 0.06316989240294787, "median": 0.06375195999862626, "min": 0.05845280599896796, "rel_spread_pct": 16.788697954223768, "stdev": 0.004077935262086413 } }, { "admitted": 11017, "cpu": { "max": 0.6318911269999999, "mean": 0.6176950376, "median": 0.623304639, "min": 0.594239489, "rel_spread_pct": 6.040647805927838, "stdev": 0.015817213846896022 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_rand_100000", "overflow": 0, "peak_rss_kib_max": 134048, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.5942861869989429, 0.6318963819940109, 0.6233428139967145, 0.6092818469915073, 0.6298116919933818 ], "setup_s_median": 0.022057192007196136, "wall": { "max": 0.6318963819940109, "mean": 0.6177237843949115, "median": 0.6233428139967145, "min": 0.5942861869989429, "rel_spread_pct": 6.033629352991346, "stdev": 0.01580459431067121 } }, { "admitted": 11017, "cpu": { "max": 0.10611933600000001, "mean": 0.086824469, "median": 0.08601087099999993, "min": 0.07635693600000004, "rel_spread_pct": 34.603067791279535, "stdev": 0.011743099469433423 }, "delivered": 11017, "dropped": 388983, "name": "cls_setsize_rand_1000000", "overflow": 0, "peak_rss_kib_max": 244620, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.07635344300069846, 0.08600624100654386, 0.07851348500116728, 0.10611291199165862, 0.08710759499808773 ], "setup_s_median": 0.2605301010044059, "wall": { "max": 0.10611291199165862, "mean": 0.0868187351996312, "median": 0.08600624100654386, "min": 0.07635344300069846, "rel_spread_pct": 34.601522683331645, "stdev": 0.011742107631799985 } }, { "admitted": 11017, "cpu": { "max": 0.904845492, "mean": 0.690935202, "median": 0.6544972190000001, "min": 0.609503935, "rel_spread_pct": 45.124952165763126, "stdev": 0.12149079092960213 }, "delivered": 400000, "dropped": 388983, "name": "py_setsize_rand_1000000", "overflow": 0, "peak_rss_kib_max": 244636, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.9279646139912074, 0.6095059249928454, 0.6240621460019611, 0.6618013650004286, 0.6545001990016317 ], "setup_s_median": 0.2692041319969576, "wall": { "max": 0.9279646139912074, "mean": 0.6955668497976149, "median": 0.6545001990016317, "min": 0.6095059249928454, "rel_spread_pct": 48.656774968767905, "stdev": 0.13167640640650594 } }, { "admitted": 400000, "cpu": { "max": 0.06820810199999999, "mean": 0.06554207199999999, "median": 0.064635893, "min": 0.06314112300000002, "rel_spread_pct": 7.839265096871133, "stdev": 0.0023477257618925397 }, "delivered": 0, "dropped": 0, "name": "buf_admit_all", "overflow": 0, "peak_rss_kib_max": 286744, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.06313545900047757, 0.06463035898923408, 0.06383686899789609, 0.06792326799768489, 0.06820326599699911 ], "setup_s_median": 0.053027529997052625, "wall": { "max": 0.06820326599699911, "mean": 0.06554584419645834, "median": 0.06463035898923408, "min": 0.06313545900047757, "rel_spread_pct": 7.8412174646371975, "stdev": 0.002360227782629169 } }, { "admitted": 400000, "cpu": { "max": 0.904804694, "mean": 0.8439399242, "median": 0.8309639899999999, "min": 0.8105430670000001, "rel_spread_pct": 11.343647635079813, "stdev": 0.038641571123391165 }, "delivered": 400000, "dropped": 0, "name": "py_buf_admit_all", "overflow": 0, "peak_rss_kib_max": 286860, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.8106346249987837, 0.8309851960075321, 0.815771608002251, 0.9048149049922358, 0.8577083849959308 ], "setup_s_median": 0.05332791600085329, "wall": { "max": 0.9048149049922358, "mean": 0.8439829437993467, "median": 0.8309851960075321, "min": 0.8106346249987837, "rel_spread_pct": 11.333568930703127, "stdev": 0.03861944626918593 } }, { "admitted": 40280, "cpu": { "max": 0.068704455, "mean": 0.06125400760000001, "median": 0.05944569300000002, "min": 0.053433707, "rel_spread_pct": 25.688569229060874, "stdev": 0.005959073028260423 }, "delivered": 0, "dropped": 359720, "name": "buf_addrset_10pct", "overflow": 0, "peak_rss_kib_max": 286712, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.059440797005663626, 0.0534295350080356, 0.06550228799460456, 0.06869784400623757, 0.05950768799812067 ], "setup_s_median": 0.057063827000092715, "wall": { "max": 0.06869784400623757, "mean": 0.061315630402532406, "median": 0.05950768799812067, "min": 0.0534295350080356, "rel_spread_pct": 25.65770829255569, "stdev": 0.005937157065018016 } }, { "admitted": 40280, "cpu": { "max": 0.688719408, "mean": 0.65089942, "median": 0.6478121189999999, "min": 0.622983514, "rel_spread_pct": 10.147370213060189, "stdev": 0.02481256423581017 }, "delivered": 400000, "dropped": 359720, "name": "py_buf_addrset_10pct", "overflow": 0, "peak_rss_kib_max": 286888, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.6230019400099991, 0.6369989200029522, 0.6887509379885159, 0.6579965470009483, 0.6484530190064106 ], "setup_s_median": 0.05583993600157555, "wall": { "max": 0.6887509379885159, "mean": 0.6510402728017652, "median": 0.6484530190064106, "min": 0.6230019400099991, "rel_spread_pct": 10.139361842938209, "stdev": 0.024801641054200286 } }, { "admitted": 64533, "cpu": { "max": 0.10745571000000004, "mean": 0.0806197188, "median": 0.07182598600000001, "min": 0.06598056299999999, "rel_spread_pct": 57.74392989189183, "stdev": 0.01673714896509078 }, "delivered": 0, "dropped": 335467, "name": "buf_flowheads_3", "overflow": 0, "peak_rss_kib_max": 295056, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.0659868259972427, 0.0860852689947933, 0.07184485600737389, 0.10746121499687433, 0.07173902999784332 ], "setup_s_median": 0.055291103999479674, "wall": { "max": 0.10746121499687433, "mean": 0.0806234391988255, "median": 0.07184485600737389, "min": 0.0659868259972427, "rel_spread_pct": 57.72770843242396, "stdev": 0.016735624355091586 } }, { "admitted": 61825, "cpu": { "max": 0.919909912, "mean": 0.8742304532, "median": 0.8681407640000001, "min": 0.8368072690000001, "rel_spread_pct": 9.572484837263087, "stdev": 0.032344642319588154 }, "delivered": 400000, "dropped": 338175, "name": "py_buf_flowheads_3", "overflow": 0, "peak_rss_kib_max": 291316, "processed": 400000, "reps": 5, "samples_wall_s": [ 0.8681653659878066, 0.8550433700002031, 0.9199766759993508, 0.8913345990004018, 0.8368383829947561 ], "setup_s_median": 0.055304227003944106, "wall": { "max": 0.9199766759993508, "mean": 0.8742716787965037, "median": 0.8681653659878066, "min": 0.8368383829947561, "rel_spread_pct": 9.576319934163605, "stdev": 0.03235875679418765 } } ], "scenarios": [ { "expect": 400000, "group": "raw", "kind": "classic_next", "name": "raw_next", "note": "one Python call + 2 objects per packet", "params": {} }, { "expect": 400000, "group": "raw", "kind": "classic_loop", "name": "raw_loop", "note": "one callback + 2 objects per packet", "params": {} }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_32_meta", "note": "metadata parsed, packet bytes not sliced out", "params": { "batch": 32, "slice": false } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_32_slice", "note": "one bytes object per packet, like next()", "params": { "batch": 32, "slice": true } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_128_meta", "note": "metadata parsed, packet bytes not sliced out", "params": { "batch": 128, "slice": false } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_128_slice", "note": "one bytes object per packet, like next()", "params": { "batch": 128, "slice": true } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_512_meta", "note": "metadata parsed, packet bytes not sliced out", "params": { "batch": 512, "slice": false } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_512_slice", "note": "one bytes object per packet, like next()", "params": { "batch": 512, "slice": true } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_1024_meta", "note": "metadata parsed, packet bytes not sliced out", "params": { "batch": 1024, "slice": false } }, { "expect": 400000, "group": "raw", "kind": "next_batch", "name": "raw_next_batch_1024_slice", "note": "one bytes object per packet, like next()", "params": { "batch": 1024, "slice": true } }, { "expect": 400000, "group": "classify", "kind": "filtered", "name": "cls_admit_all", "pair": "py_admit_all", "params": { "mask": 7 } }, { "expect": 400000, "group": "classify", "kind": "py_admit_all", "name": "py_admit_all", "params": {} }, { "expect": 200021, "group": "classify", "kind": "filtered", "name": "cls_addrset_50pct", "note": "50.01% of packets admitted", "pair": "py_addrset_50pct", "params": { "mask": 4, "set": "50pct" } }, { "expect": 200021, "group": "classify", "kind": "py_addrset", "name": "py_addrset_50pct", "params": { "set": "50pct" } }, { "expect": 40280, "group": "classify", "kind": "filtered", "name": "cls_addrset_10pct", "note": "10.07% of packets admitted", "pair": "py_addrset_10pct", "params": { "mask": 4, "set": "10pct" } }, { "expect": 40280, "group": "classify", "kind": "py_addrset", "name": "py_addrset_10pct", "params": { "set": "10pct" } }, { "expect": 5046, "group": "classify", "kind": "filtered", "name": "cls_addrset_1pct", "note": "1.26% of packets admitted", "pair": "py_addrset_1pct", "params": { "mask": 4, "set": "1pct" } }, { "expect": 5046, "group": "classify", "kind": "py_addrset", "name": "py_addrset_1pct", "params": { "set": "1pct" } }, { "expect": 0, "group": "classify", "kind": "filtered", "name": "cls_addrset_0pct", "note": "0.00% of packets admitted", "pair": "py_addrset_0pct", "params": { "mask": 4, "set": "0pct" } }, { "expect": 0, "group": "classify", "kind": "py_addrset", "name": "py_addrset_0pct", "params": { "set": "0pct" } }, { "expect": 0, "group": "classify", "kind": "filtered", "name": "cls_admit_none", "note": "classify everything, deliver nothing", "pair": "py_admit_none", "params": { "mask": 0 } }, { "expect": 0, "group": "classify", "kind": "py_admit_none", "name": "py_admit_none", "params": {} }, { "expect": 23620, "expect_mode": "min", "group": "classify", "kind": "filtered", "name": "cls_flowheads_1", "note": "C table is direct-mapped with eviction; admits >= exact count", "pair": "py_flowheads_1", "params": { "cutoff": 1, "mask": 2 } }, { "expect": 23620, "group": "classify", "kind": "py_flowheads", "name": "py_flowheads_1", "params": { "cutoff": 1 } }, { "expect": 61825, "expect_mode": "min", "group": "classify", "kind": "filtered", "name": "cls_flowheads_3", "note": "C table is direct-mapped with eviction; admits >= exact count", "pair": "py_flowheads_3", "params": { "cutoff": 3, "mask": 2 } }, { "expect": 61825, "group": "classify", "kind": "py_flowheads", "name": "py_flowheads_3", "params": { "cutoff": 3 } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_seq_10", "note": "10 addresses, consecutive padding, 2.75% admitted", "pair": "py_setsize_seq_10", "params": { "mask": 4, "sizeset": "10" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_seq_10", "params": { "sizeset": "10" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_seq_1000", "note": "1000 addresses, consecutive padding, 2.75% admitted", "pair": "py_setsize_seq_1000", "params": { "mask": 4, "sizeset": "1000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_seq_1000", "params": { "sizeset": "1000" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_seq_100000", "note": "100000 addresses, consecutive padding, 2.75% admitted", "pair": "py_setsize_seq_100000", "params": { "mask": 4, "sizeset": "100000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_seq_100000", "params": { "sizeset": "100000" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_seq_1000000", "note": "1000000 addresses, consecutive padding, 2.75% admitted", "pair": "py_setsize_seq_1000000", "params": { "mask": 4, "sizeset": "1000000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_seq_1000000", "params": { "sizeset": "1000000" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_rand_10", "note": "10 addresses, random padding, 2.75% admitted", "pair": "py_setsize_rand_10", "params": { "mask": 4, "sizeset_rand": "10" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_rand_10", "params": { "sizeset_rand": "10" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_rand_1000", "note": "1000 addresses, random padding, 2.75% admitted", "pair": "py_setsize_rand_1000", "params": { "mask": 4, "sizeset_rand": "1000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_rand_1000", "params": { "sizeset_rand": "1000" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_rand_100000", "note": "100000 addresses, random padding, 2.75% admitted", "pair": "py_setsize_rand_100000", "params": { "mask": 4, "sizeset_rand": "100000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_rand_100000", "params": { "sizeset_rand": "100000" } }, { "expect": 11017, "group": "classify", "kind": "filtered", "name": "cls_setsize_rand_1000000", "note": "1000000 addresses, random padding, 2.75% admitted", "pair": "py_setsize_rand_1000000", "params": { "mask": 4, "sizeset_rand": "1000000" } }, { "expect": 11017, "group": "classify", "kind": "py_setsize", "name": "py_setsize_rand_1000000", "params": { "sizeset_rand": "1000000" } }, { "expect": 400000, "group": "buffer", "kind": "to_buffer", "name": "buf_admit_all", "pair": "py_buf_admit_all", "params": { "mask": 7 } }, { "expect": 400000, "group": "buffer", "kind": "py_buf_all", "name": "py_buf_admit_all", "params": {} }, { "expect": 40280, "group": "buffer", "kind": "to_buffer", "name": "buf_addrset_10pct", "pair": "py_buf_addrset_10pct", "params": { "mask": 4, "set": "10pct" } }, { "expect": 40280, "group": "buffer", "kind": "py_buf_addrset", "name": "py_buf_addrset_10pct", "params": { "set": "10pct" } }, { "expect": 61825, "expect_mode": "min", "group": "buffer", "kind": "to_buffer", "name": "buf_flowheads_3", "note": "C table is direct-mapped with eviction; admits >= exact count", "pair": "py_buf_flowheads_3", "params": { "cutoff": 3, "mask": 2 } }, { "expect": 61825, "group": "buffer", "kind": "py_buf_flowheads", "name": "py_buf_flowheads_3", "params": { "cutoff": 3 } } ], "system": { "capture": { "avg_packet_size": 443.3, "bytes": 177313087, "distinct_flow_keys": 23620, "file_size": 183713111, "ipv4_packets": 388066, "ipv6_packets": 11934, "packets": 400000, "path": "/home/stamparm/Private/Work/pcapy-ng/benchmarks/data/bench.pcap", "vlan_packets": 7860 }, "cpu": "AMD Ryzen 7 PRO 4750U with Radeon Graphics", "cpu_count": 16, "cpu_freq_policy": { "scaling_driver": "acpi-cpufreq", "scaling_governor": "schedutil", "scaling_max_freq_khz": 1700000 }, "kernel": "6.8.0-137-generic", "libpcap": { "path": "/usr/lib/x86_64-linux-gnu/libpcap.so.1.10.4", "version": "libpcap version 1.10.4 (with TPACKET_V3)" }, "loadavg_at_start": [ 2.88330078125, 2.2626953125, 1.94677734375 ], "pcapy_module": "/tmp/claude-1000/-home-stamparm-Private-Work-pcapy-ng/d862c1ef-6d01-498b-a4a8-be1665f1e781/scratchpad/v/lib/python3.12/site-packages/pcapy.cpython-312-x86_64-linux-gnu.so", "pcapy_ng_git": "639a6a7-dirty", "pcapy_ng_version": "2.0.2", "platform": "Linux-6.8.0-137-generic-x86_64-with-glibc2.39", "python": "3.12.3", "python_executable": "/tmp/claude-1000/-home-stamparm-Private-Work-pcapy-ng/d862c1ef-6d01-498b-a4a8-be1665f1e781/scratchpad/v/bin/python", "python_implementation": "CPython" }, "table": [ { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.234391, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one Python call + 2 objects per packet", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 1706360, "pkts_processed_per_s": 1706360, "processed": 400000, "reps": 5, "scenario": "raw_next", "setup_median_s": 5.3e-05, "wall_max_s": 0.246089, "wall_median_s": 0.234417, "wall_min_s": 0.228871, "wall_spread_pct": 7.35, "wall_stdev_s": 0.007109 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.142987, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one callback + 2 objects per packet", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2796969, "pkts_processed_per_s": 2796969, "processed": 400000, "reps": 5, "scenario": "raw_loop", "setup_median_s": 4.5e-05, "wall_max_s": 0.217811, "wall_median_s": 0.143012, "wall_min_s": 0.133312, "wall_spread_pct": 59.08, "wall_stdev_s": 0.03532 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.149237, "delivered": 0, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "metadata parsed, packet bytes not sliced out", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 2680086, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_32_meta", "setup_median_s": 5.5e-05, "wall_max_s": 0.220623, "wall_median_s": 0.149249, "wall_min_s": 0.144879, "wall_spread_pct": 50.75, "wall_stdev_s": 0.032648 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.19693, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one bytes object per packet, like next()", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2030878, "pkts_processed_per_s": 2030878, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_32_slice", "setup_median_s": 5.6e-05, "wall_max_s": 0.205797, "wall_median_s": 0.196959, "wall_min_s": 0.191706, "wall_spread_pct": 7.15, "wall_stdev_s": 0.006396 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.140892, "delivered": 0, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "metadata parsed, packet bytes not sliced out", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 2839053, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_128_meta", "setup_median_s": 4.5e-05, "wall_max_s": 0.156363, "wall_median_s": 0.140892, "wall_min_s": 0.134778, "wall_spread_pct": 15.32, "wall_stdev_s": 0.008051 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.188201, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one bytes object per packet, like next()", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2124959, "pkts_processed_per_s": 2124959, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_128_slice", "setup_median_s": 5.1e-05, "wall_max_s": 0.192286, "wall_median_s": 0.188239, "wall_min_s": 0.183372, "wall_spread_pct": 4.74, "wall_stdev_s": 0.004051 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.140439, "delivered": 0, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "metadata parsed, packet bytes not sliced out", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 2847134, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_512_meta", "setup_median_s": 5.5e-05, "wall_max_s": 0.144643, "wall_median_s": 0.140492, "wall_min_s": 0.136585, "wall_spread_pct": 5.74, "wall_stdev_s": 0.002856 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.190855, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one bytes object per packet, like next()", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2095635, "pkts_processed_per_s": 2095635, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_512_slice", "setup_median_s": 5.9e-05, "wall_max_s": 0.192626, "wall_median_s": 0.190873, "wall_min_s": 0.173698, "wall_spread_pct": 9.92, "wall_stdev_s": 0.00875 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.142377, "delivered": 0, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "metadata parsed, packet bytes not sliced out", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 2807836, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_1024_meta", "setup_median_s": 4.6e-05, "wall_max_s": 0.161361, "wall_median_s": 0.142458, "wall_min_s": 0.138157, "wall_spread_pct": 16.29, "wall_stdev_s": 0.009501 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.194178, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "raw", "note": "one bytes object per packet, like next()", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2059773, "pkts_processed_per_s": 2059773, "processed": 400000, "reps": 5, "scenario": "raw_next_batch_1024_slice", "setup_median_s": 4.6e-05, "wall_max_s": 0.217089, "wall_median_s": 0.194196, "wall_min_s": 0.187767, "wall_spread_pct": 15.1, "wall_stdev_s": 0.012283 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.152446, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "classify", "note": "", "overflow": 0, "pair": "py_admit_all", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 2623516, "pkts_processed_per_s": 2623516, "processed": 400000, "reps": 5, "scenario": "cls_admit_all", "setup_median_s": 4.8e-05, "wall_max_s": 0.174407, "wall_median_s": 0.152467, "wall_min_s": 0.139351, "wall_spread_pct": 22.99, "wall_stdev_s": 0.013275 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.562985, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 710438, "pkts_processed_per_s": 710438, "processed": 400000, "reps": 5, "scenario": "py_admit_all", "setup_median_s": 5.1e-05, "wall_max_s": 0.61106, "wall_median_s": 0.563033, "wall_min_s": 0.557265, "wall_spread_pct": 9.55, "wall_stdev_s": 0.022689 }, { "admitted": 200021, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.114602, "delivered": 200021, "dropped": 199979, "expect_mode": "exact", "expected_admitted": 200021, "group": "classify", "note": "50.01% of packets admitted", "overflow": 0, "pair": "py_addrset_50pct", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 1745107, "pkts_processed_per_s": 3489847, "processed": 400000, "reps": 5, "scenario": "cls_addrset_50pct", "setup_median_s": 0.000134, "wall_max_s": 0.123983, "wall_median_s": 0.114618, "wall_min_s": 0.109596, "wall_spread_pct": 12.55, "wall_stdev_s": 0.005658 }, { "admitted": 200021, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.600157, "delivered": 400000, "dropped": 199979, "expect_mode": "exact", "expected_admitted": 200021, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 666400, "pkts_processed_per_s": 666400, "processed": 400000, "reps": 5, "scenario": "py_addrset_50pct", "setup_median_s": 0.000132, "wall_max_s": 0.630193, "wall_median_s": 0.60024, "wall_min_s": 0.586327, "wall_spread_pct": 7.31, "wall_stdev_s": 0.018538 }, { "admitted": 40280, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.069106, "delivered": 40280, "dropped": 359720, "expect_mode": "exact", "expected_admitted": 40280, "group": "classify", "note": "10.07% of packets admitted", "overflow": 0, "pair": "py_addrset_10pct", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 582674, "pkts_processed_per_s": 5786240, "processed": 400000, "reps": 5, "scenario": "cls_addrset_10pct", "setup_median_s": 7.8e-05, "wall_max_s": 0.076041, "wall_median_s": 0.06913, "wall_min_s": 0.062756, "wall_spread_pct": 19.22, "wall_stdev_s": 0.005915 }, { "admitted": 40280, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.607631, "delivered": 400000, "dropped": 359720, "expect_mode": "exact", "expected_admitted": 40280, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 658234, "pkts_processed_per_s": 658234, "processed": 400000, "reps": 5, "scenario": "py_addrset_10pct", "setup_median_s": 8.7e-05, "wall_max_s": 0.61629, "wall_median_s": 0.607686, "wall_min_s": 0.601458, "wall_spread_pct": 2.44, "wall_stdev_s": 0.006225 }, { "admitted": 5046, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.054891, "delivered": 5046, "dropped": 394954, "expect_mode": "exact", "expected_admitted": 5046, "group": "classify", "note": "1.26% of packets admitted", "overflow": 0, "pair": "py_addrset_1pct", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 91887, "pkts_processed_per_s": 7283987, "processed": 400000, "reps": 5, "scenario": "cls_addrset_1pct", "setup_median_s": 6.1e-05, "wall_max_s": 0.056365, "wall_median_s": 0.054915, "wall_min_s": 0.049615, "wall_spread_pct": 12.29, "wall_stdev_s": 0.002642 }, { "admitted": 5046, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.592855, "delivered": 400000, "dropped": 394954, "expect_mode": "exact", "expected_admitted": 5046, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 674658, "pkts_processed_per_s": 674658, "processed": 400000, "reps": 5, "scenario": "py_addrset_1pct", "setup_median_s": 7.8e-05, "wall_max_s": 0.660835, "wall_median_s": 0.592892, "wall_min_s": 0.590223, "wall_spread_pct": 11.91, "wall_stdev_s": 0.0307 }, { "admitted": 0, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.058413, "delivered": 0, "dropped": 400000, "expect_mode": "exact", "expected_admitted": 0, "group": "classify", "note": "0.00% of packets admitted", "overflow": 0, "pair": "py_addrset_0pct", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 6844751, "processed": 400000, "reps": 5, "scenario": "cls_addrset_0pct", "setup_median_s": 0.000195, "wall_max_s": 0.068999, "wall_median_s": 0.058439, "wall_min_s": 0.056501, "wall_spread_pct": 21.39, "wall_stdev_s": 0.005215 }, { "admitted": 0, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.606344, "delivered": 400000, "dropped": 400000, "expect_mode": "exact", "expected_admitted": 0, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 659623, "pkts_processed_per_s": 659623, "processed": 400000, "reps": 5, "scenario": "py_addrset_0pct", "setup_median_s": 0.000242, "wall_max_s": 0.646391, "wall_median_s": 0.606406, "wall_min_s": 0.598403, "wall_spread_pct": 7.91, "wall_stdev_s": 0.022317 }, { "admitted": 0, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.049454, "delivered": 0, "dropped": 400000, "expect_mode": "exact", "expected_admitted": 0, "group": "classify", "note": "classify everything, deliver nothing", "overflow": 0, "pair": "py_admit_none", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 8085182, "processed": 400000, "reps": 5, "scenario": "cls_admit_none", "setup_median_s": 5e-05, "wall_max_s": 0.056014, "wall_median_s": 0.049473, "wall_min_s": 0.048292, "wall_spread_pct": 15.61, "wall_stdev_s": 0.00385 }, { "admitted": 0, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.565804, "delivered": 400000, "dropped": 400000, "expect_mode": "exact", "expected_admitted": 0, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 706890, "pkts_processed_per_s": 706890, "processed": 400000, "reps": 5, "scenario": "py_admit_none", "setup_median_s": 4.7e-05, "wall_max_s": 0.617344, "wall_median_s": 0.565859, "wall_min_s": 0.543316, "wall_spread_pct": 13.08, "wall_stdev_s": 0.032459 }, { "admitted": 25477, "admitted_delta_pct": 7.862, "admitted_ok": true, "cpu_median_s": 0.087905, "delivered": 25477, "dropped": 374523, "expect_mode": "min", "expected_admitted": 23620, "group": "classify", "note": "C table is direct-mapped with eviction; admits >= exact count", "overflow": 0, "pair": "py_flowheads_1", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 289696, "pkts_processed_per_s": 4548354, "processed": 400000, "reps": 5, "scenario": "cls_flowheads_1", "setup_median_s": 5.6e-05, "wall_max_s": 0.090835, "wall_median_s": 0.087944, "wall_min_s": 0.074143, "wall_spread_pct": 18.98, "wall_stdev_s": 0.007531 }, { "admitted": 23620, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.838304, "delivered": 400000, "dropped": 376380, "expect_mode": "exact", "expected_admitted": 23620, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 477131, "pkts_processed_per_s": 477131, "processed": 400000, "reps": 5, "scenario": "py_flowheads_1", "setup_median_s": 4.8e-05, "wall_max_s": 0.900733, "wall_median_s": 0.838343, "wall_min_s": 0.79212, "wall_spread_pct": 12.96, "wall_stdev_s": 0.044034 }, { "admitted": 64533, "admitted_delta_pct": 4.38, "admitted_ok": true, "cpu_median_s": 0.094448, "delivered": 64533, "dropped": 335467, "expect_mode": "min", "expected_admitted": 61825, "group": "classify", "note": "C table is direct-mapped with eviction; admits >= exact count", "overflow": 0, "pair": "py_flowheads_3", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 683109, "pkts_processed_per_s": 4234171, "processed": 400000, "reps": 5, "scenario": "cls_flowheads_3", "setup_median_s": 5.3e-05, "wall_max_s": 0.123834, "wall_median_s": 0.094469, "wall_min_s": 0.084386, "wall_spread_pct": 41.76, "wall_stdev_s": 0.014987 }, { "admitted": 61825, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.77899, "delivered": 400000, "dropped": 338175, "expect_mode": "exact", "expected_admitted": 61825, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 513025, "pkts_processed_per_s": 513025, "processed": 400000, "reps": 5, "scenario": "py_flowheads_3", "setup_median_s": 4.9e-05, "wall_max_s": 0.834075, "wall_median_s": 0.779688, "wall_min_s": 0.754363, "wall_spread_pct": 10.22, "wall_stdev_s": 0.031593 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.056986, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "10 addresses, consecutive padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_seq_10", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 193239, "pkts_processed_per_s": 7016063, "processed": 400000, "reps": 5, "scenario": "cls_setsize_seq_10", "setup_median_s": 6.8e-05, "wall_max_s": 0.062105, "wall_median_s": 0.057012, "wall_min_s": 0.055948, "wall_spread_pct": 10.8, "wall_stdev_s": 0.002539 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.599151, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 667579, "pkts_processed_per_s": 667579, "processed": 400000, "reps": 5, "scenario": "py_setsize_seq_10", "setup_median_s": 5.9e-05, "wall_max_s": 0.65014, "wall_median_s": 0.599179, "wall_min_s": 0.581128, "wall_spread_pct": 11.52, "wall_stdev_s": 0.027929 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.058758, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "1000 addresses, consecutive padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_seq_1000", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 187418, "pkts_processed_per_s": 6804711, "processed": 400000, "reps": 5, "scenario": "cls_setsize_seq_1000", "setup_median_s": 0.00019, "wall_max_s": 0.063411, "wall_median_s": 0.058783, "wall_min_s": 0.0582, "wall_spread_pct": 8.87, "wall_stdev_s": 0.00218 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.607368, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 658523, "pkts_processed_per_s": 658523, "processed": 400000, "reps": 5, "scenario": "py_setsize_seq_1000", "setup_median_s": 0.000176, "wall_max_s": 0.64709, "wall_median_s": 0.60742, "wall_min_s": 0.600786, "wall_spread_pct": 7.62, "wall_stdev_s": 0.019074 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.061467, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "100000 addresses, consecutive padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_seq_100000", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 179248, "pkts_processed_per_s": 6508063, "processed": 400000, "reps": 5, "scenario": "cls_setsize_seq_100000", "setup_median_s": 0.015902, "wall_max_s": 0.085658, "wall_median_s": 0.061462, "wall_min_s": 0.059566, "wall_spread_pct": 42.45, "wall_stdev_s": 0.01114 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.625533, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 639410, "pkts_processed_per_s": 639410, "processed": 400000, "reps": 5, "scenario": "py_setsize_seq_100000", "setup_median_s": 0.017381, "wall_max_s": 0.72562, "wall_median_s": 0.625576, "wall_min_s": 0.609206, "wall_spread_pct": 18.61, "wall_stdev_s": 0.047531 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.082168, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "1000000 addresses, consecutive padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_seq_1000000", "peak_rss_mib": 238.9, "pkts_delivered_per_s": 134066, "pkts_processed_per_s": 4867613, "processed": 400000, "reps": 5, "scenario": "cls_setsize_seq_1000000", "setup_median_s": 0.186716, "wall_max_s": 0.095131, "wall_median_s": 0.082176, "wall_min_s": 0.074516, "wall_spread_pct": 25.09, "wall_stdev_s": 0.007465 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.651151, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 239.0, "pkts_delivered_per_s": 614273, "pkts_processed_per_s": 614273, "processed": 400000, "reps": 5, "scenario": "py_setsize_seq_1000000", "setup_median_s": 0.19061, "wall_max_s": 0.792211, "wall_median_s": 0.651176, "wall_min_s": 0.621575, "wall_spread_pct": 26.2, "wall_stdev_s": 0.06697 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.059536, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "10 addresses, random padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_rand_10", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 184961, "pkts_processed_per_s": 6715504, "processed": 400000, "reps": 5, "scenario": "cls_setsize_rand_10", "setup_median_s": 6.1e-05, "wall_max_s": 0.064177, "wall_median_s": 0.059564, "wall_min_s": 0.053091, "wall_spread_pct": 18.61, "wall_stdev_s": 0.004661 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.593664, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 673760, "pkts_processed_per_s": 673760, "processed": 400000, "reps": 5, "scenario": "py_setsize_rand_10", "setup_median_s": 7.7e-05, "wall_max_s": 0.666971, "wall_median_s": 0.593683, "wall_min_s": 0.592251, "wall_spread_pct": 12.59, "wall_stdev_s": 0.032392 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.065292, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "1000 addresses, random padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_rand_1000", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 168672, "pkts_processed_per_s": 6124087, "processed": 400000, "reps": 5, "scenario": "cls_setsize_rand_1000", "setup_median_s": 0.000218, "wall_max_s": 0.068828, "wall_median_s": 0.065316, "wall_min_s": 0.059931, "wall_spread_pct": 13.62, "wall_stdev_s": 0.003738 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.613796, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 651382, "pkts_processed_per_s": 651382, "processed": 400000, "reps": 5, "scenario": "py_setsize_rand_1000", "setup_median_s": 0.00019, "wall_max_s": 0.666542, "wall_median_s": 0.614079, "wall_min_s": 0.593518, "wall_spread_pct": 11.89, "wall_stdev_s": 0.029363 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.063757, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "100000 addresses, random padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_rand_100000", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 172810, "pkts_processed_per_s": 6274316, "processed": 400000, "reps": 5, "scenario": "cls_setsize_rand_100000", "setup_median_s": 0.021056, "wall_max_s": 0.069156, "wall_median_s": 0.063752, "wall_min_s": 0.058453, "wall_spread_pct": 16.79, "wall_stdev_s": 0.004078 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.623305, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 130.9, "pkts_delivered_per_s": 641701, "pkts_processed_per_s": 641701, "processed": 400000, "reps": 5, "scenario": "py_setsize_rand_100000", "setup_median_s": 0.022057, "wall_max_s": 0.631896, "wall_median_s": 0.623343, "wall_min_s": 0.594286, "wall_spread_pct": 6.03, "wall_stdev_s": 0.015805 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.086011, "delivered": 11017, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "1000000 addresses, random padding, 2.75% admitted", "overflow": 0, "pair": "py_setsize_rand_1000000", "peak_rss_mib": 238.9, "pkts_delivered_per_s": 128095, "pkts_processed_per_s": 4650825, "processed": 400000, "reps": 5, "scenario": "cls_setsize_rand_1000000", "setup_median_s": 0.26053, "wall_max_s": 0.106113, "wall_median_s": 0.086006, "wall_min_s": 0.076353, "wall_spread_pct": 34.6, "wall_stdev_s": 0.011742 }, { "admitted": 11017, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.654497, "delivered": 400000, "dropped": 388983, "expect_mode": "exact", "expected_admitted": 11017, "group": "classify", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 238.9, "pkts_delivered_per_s": 611153, "pkts_processed_per_s": 611153, "processed": 400000, "reps": 5, "scenario": "py_setsize_rand_1000000", "setup_median_s": 0.269204, "wall_max_s": 0.927965, "wall_median_s": 0.6545, "wall_min_s": 0.609506, "wall_spread_pct": 48.66, "wall_stdev_s": 0.131676 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.064636, "delivered": 0, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "buffer", "note": "", "overflow": 0, "pair": "py_buf_admit_all", "peak_rss_mib": 280.0, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 6189041, "processed": 400000, "reps": 5, "scenario": "buf_admit_all", "setup_median_s": 0.053028, "wall_max_s": 0.068203, "wall_median_s": 0.06463, "wall_min_s": 0.063135, "wall_spread_pct": 7.84, "wall_stdev_s": 0.00236 }, { "admitted": 400000, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.830964, "delivered": 400000, "dropped": 0, "expect_mode": "exact", "expected_admitted": 400000, "group": "buffer", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 280.1, "pkts_delivered_per_s": 481356, "pkts_processed_per_s": 481356, "processed": 400000, "reps": 5, "scenario": "py_buf_admit_all", "setup_median_s": 0.053328, "wall_max_s": 0.904815, "wall_median_s": 0.830985, "wall_min_s": 0.810635, "wall_spread_pct": 11.33, "wall_stdev_s": 0.038619 }, { "admitted": 40280, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.059446, "delivered": 0, "dropped": 359720, "expect_mode": "exact", "expected_admitted": 40280, "group": "buffer", "note": "", "overflow": 0, "pair": "py_buf_addrset_10pct", "peak_rss_mib": 280.0, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 6721820, "processed": 400000, "reps": 5, "scenario": "buf_addrset_10pct", "setup_median_s": 0.057064, "wall_max_s": 0.068698, "wall_median_s": 0.059508, "wall_min_s": 0.05343, "wall_spread_pct": 25.66, "wall_stdev_s": 0.005937 }, { "admitted": 40280, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.647812, "delivered": 400000, "dropped": 359720, "expect_mode": "exact", "expected_admitted": 40280, "group": "buffer", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 280.2, "pkts_delivered_per_s": 616852, "pkts_processed_per_s": 616852, "processed": 400000, "reps": 5, "scenario": "py_buf_addrset_10pct", "setup_median_s": 0.05584, "wall_max_s": 0.688751, "wall_median_s": 0.648453, "wall_min_s": 0.623002, "wall_spread_pct": 10.14, "wall_stdev_s": 0.024802 }, { "admitted": 64533, "admitted_delta_pct": 4.38, "admitted_ok": true, "cpu_median_s": 0.071826, "delivered": 0, "dropped": 335467, "expect_mode": "min", "expected_admitted": 61825, "group": "buffer", "note": "C table is direct-mapped with eviction; admits >= exact count", "overflow": 0, "pair": "py_buf_flowheads_3", "peak_rss_mib": 288.1, "pkts_delivered_per_s": 0, "pkts_processed_per_s": 5567552, "processed": 400000, "reps": 5, "scenario": "buf_flowheads_3", "setup_median_s": 0.055291, "wall_max_s": 0.107461, "wall_median_s": 0.071845, "wall_min_s": 0.065987, "wall_spread_pct": 57.73, "wall_stdev_s": 0.016736 }, { "admitted": 61825, "admitted_delta_pct": 0.0, "admitted_ok": true, "cpu_median_s": 0.868141, "delivered": 400000, "dropped": 338175, "expect_mode": "exact", "expected_admitted": 61825, "group": "buffer", "note": "", "overflow": 0, "pair": "", "peak_rss_mib": 284.5, "pkts_delivered_per_s": 460741, "pkts_processed_per_s": 460741, "processed": 400000, "reps": 5, "scenario": "py_buf_flowheads_3", "setup_median_s": 0.055304, "wall_max_s": 0.919977, "wall_median_s": 0.868165, "wall_min_s": 0.836838, "wall_spread_pct": 9.58, "wall_stdev_s": 0.032359 } ] }pcapy-ng-2.1.0/bpfobj.cc000077500000000000000000000162761524067405300150260ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #define PY_SSIZE_T_CLEAN #include #include #include "bpfobj.h" #include "pcapy.h" // internal bpfobject typedef struct { PyObject_HEAD struct bpf_program bpf; } bpfobject; // BPFProgramType static void bpfprog_dealloc(bpfobject* bpf) { // Fix: Removed #ifndef WIN32. Always free the BPF code. pcap_freecode(&bpf->bpf); PyObject_Del(bpf); } PyObject* BPFError; // BPFProgram methods static PyObject* p_filter(bpfobject* bpf, PyObject* args); static PyObject* p_get_bpf(bpfobject* bpf, PyObject* args); static PyObject* p_new_bpfobject(PyTypeObject *type, PyObject* args, PyObject *kwags); static PyMethodDef bpf_methods[] = { {"filter", (PyCFunction) p_filter, METH_VARARGS, "filter(packet) applies the filter to the packet, returns 0 if there's no match"}, {"get_bpf", (PyCFunction) p_get_bpf, METH_NOARGS, "return packet-matching code as decimal numbers"}, {NULL, NULL} /* sentinel */ }; static PyObject* bpfprog_getattr(bpfobject* pp, char* name) { #if PY_MAJOR_VERSION >= 3 PyObject *nameobj = PyUnicode_FromString(name); PyObject *attr = PyObject_GenericGetAttr((PyObject *)pp, nameobj); Py_DECREF(nameobj); return attr; #else return Py_FindMethod(bpf_methods, (PyObject*)pp, name); #endif } PyTypeObject BPFProgramType = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(&PyType_Type, 0) "BPFProgram", /* tp_name */ sizeof(bpfobject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)bpfprog_dealloc, /* tp_dealloc */ 0, /* tp_print */ (getattrfunc)bpfprog_getattr, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ "BPF Program Wrapper", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ bpf_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ p_new_bpfobject /* tp_new */ #else PyObject_HEAD_INIT(NULL) 0, "BPFProgram", sizeof(bpfobject), 0, /* methods */ (destructor)bpfprog_dealloc, /* tp_dealloc*/ 0, /* tp_print*/ (getattrfunc)bpfprog_getattr, /* tp_getattr*/ 0, /* tp_setattr*/ 0, /*tp_compare*/ 0, /*tp_repr*/ 0, /*tp_as_number*/ 0, /*tp_as_sequence*/ 0, /*tp_as_mapping*/ 0, /*tp_hash */ 0, /*tp_call*/ 0, /*tp_str*/ 0, /*tp_getattro*/ 0, /*tp_setattro*/ 0, /*tp_as_buffer*/ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE, /*tp_flags*/ "BPF Program Wrapper", /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ bpf_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ p_new_bpfobject /* tp_new */ #endif }; PyObject* new_bpfobject(const struct bpf_program &bpfprog) { if (PyType_Ready(&BPFProgramType) < 0) return NULL; bpfobject *bpf; bpf = PyObject_New(bpfobject, &BPFProgramType); if (bpf == NULL) { PyErr_SetString(BPFError, "Failed to create object"); return NULL; } bpf->bpf = bpfprog; return (PyObject*)bpf; } static PyObject* p_new_bpfobject(PyTypeObject *type, PyObject *args, PyObject *kwds) { char *filter_string; int linktype = 1; // DLT_EN10MB if (!PyArg_ParseTuple(args, "s|i", &filter_string, &linktype)){ return NULL; } struct bpf_program bpfprog; if (pcap_compile_nopcap((1<<16), linktype, &bpfprog, filter_string, 0, 0)){ PyErr_SetString(BPFError, "Couldn't compile BPF program"); return NULL; } return new_bpfobject(bpfprog); } static PyObject* p_filter(bpfobject* bpf, PyObject* args) { int status; u_char* packet; Py_ssize_t len; if (Py_TYPE(bpf) != &BPFProgramType) { PyErr_SetString(BPFError, "Not a bpfprogram object"); return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args,"y#:filter",&packet, &len)){ return NULL; } #else if (!PyArg_ParseTuple(args,"s#:filter",&packet, &len)){ return NULL; } #endif status = bpf_filter(bpf->bpf.bf_insns, packet, len, len); return Py_BuildValue("i", status); } static PyObject* p_get_bpf(bpfobject* bpf, PyObject* args) { struct bpf_insn *insn; int i; int n = bpf->bpf.bf_len; PyObject* list; PyObject* instruction; insn = bpf->bpf.bf_insns; if (Py_TYPE(bpf) != &BPFProgramType) { PyErr_SetString(BPFError, "Not a bpfprogram object"); return NULL; } list = PyList_New(n); if (!list) { return NULL; } for (i = 0; i < n; ++insn, ++i) { instruction = Py_BuildValue("IIII", insn->code, insn->jt, insn->jf, insn->k); if (!instruction) { Py_DECREF(list); return NULL; } PyList_SET_ITEM(list, i, instruction); } return list; } pcapy-ng-2.1.0/bpfobj.h000077500000000000000000000007271524067405300146620ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #ifndef __bpfobj__ #define __bpfobj__ PyObject* new_bpfobject(const struct bpf_program &bpf); extern PyObject* BPFError; extern PyTypeObject BPFProgramType; extern PyTypeObject Pcaptype; #endif // __bpfobj__ pcapy-ng-2.1.0/examples/000077500000000000000000000000001524067405300150545ustar00rootroot00000000000000pcapy-ng-2.1.0/examples/01_sniff_live.py000066400000000000000000000020351524067405300200520ustar00rootroot00000000000000#!/usr/bin/env python """Classic live capture: open a device, set a BPF filter, print a line per packet. sudo python 01_sniff_live.py eth0 "tcp port 80 or udp port 53" (Live capture needs privileges; on Linux that means root or CAP_NET_RAW.) """ import sys import pcapy SNAPLEN = 65535 PROMISC = True TIMEOUT_MS = 100 def main(): dev = sys.argv[1] if len(sys.argv) > 1 else pcapy.lookupdev() bpf = sys.argv[2] if len(sys.argv) > 2 else None cap = pcapy.open_live(dev, SNAPLEN, PROMISC, TIMEOUT_MS) if bpf: cap.setfilter(bpf) print("listening on %s%s ... (Ctrl-C to stop)" % (dev, " [%s]" % bpf if bpf else "")) n = 0 while True: try: hdr, data = cap.next() except pcapy.PcapError: continue if hdr is None: continue sec, usec = hdr.getts() n += 1 print("#%d %d.%06d wire=%d cap=%d" % (n, sec, usec, hdr.getlen(), len(data))) if __name__ == "__main__": try: main() except KeyboardInterrupt: pass pcapy-ng-2.1.0/examples/02_read_pcap.py000066400000000000000000000014201524067405300176420ustar00rootroot00000000000000#!/usr/bin/env python """Read a saved capture and summarize it (no privileges needed). python 02_read_pcap.py capture.pcap """ import sys import pcapy def main(path): cap = pcapy.open_offline(path) print("datalink: %d" % cap.datalink()) count = 0 octets = 0 cap_truncated = 0 while True: hdr, data = cap.next() if hdr is None: break count += 1 octets += hdr.getlen() if hdr.getcaplen() < hdr.getlen(): cap_truncated += 1 print("packets : %d" % count) print("bytes : %d (wire)" % octets) print("snaplen-truncated: %d" % cap_truncated) if __name__ == "__main__": if len(sys.argv) < 2: sys.exit("usage: python 02_read_pcap.py ") main(sys.argv[1]) pcapy-ng-2.1.0/examples/03_filter_in_c.py000066400000000000000000000033251524067405300202100ustar00rootroot00000000000000#!/usr/bin/env python """loop_filtered: classify each packet in C, hand Python only what you ask for, drop the rest without building a Python object. python 03_filter_in_c.py capture.pcap sudo python 03_filter_in_c.py eth0 Here we keep the start of every connection (FLOW_HEAD) and anything to/from a watch list (SET_MATCH), and drop the rest (OTHER) in C. At the end we print the class counts and how much never had to cross into Python. """ import os import sys import socket import pcapy OTHER, FLOW_HEAD, SET_MATCH = 0, 1, 2 NAMES = ["OTHER", "FLOW_HEAD", "SET_MATCH"] WATCH = ["8.8.8.8", "1.1.1.1"] # any address list; can be huge ADDR_SET = b"".join(socket.inet_aton(ip) for ip in WATCH) def open_source(arg): if arg and os.path.exists(arg): return pcapy.open_offline(arg) return pcapy.open_live(arg or pcapy.lookupdev(), 65535, True, 100) def main(): cap = open_source(sys.argv[1] if len(sys.argv) > 1 else None) shown = [0] def on_packet(hdr, data, cls): if shown[0] < 20: print("keep %-9s len=%d" % (NAMES[cls], len(data))) shown[0] += 1 admit = (1 << FLOW_HEAD) | (1 << SET_MATCH) # drop OTHER in C res = cap.loop_filtered(-1, on_packet, admit, ADDR_SET, 3) admitted, dropped, n_other, n_head, n_set = res total = admitted + dropped print("\n--- summary ---") print("classes : other=%d flow_head=%d set_match=%d" % (n_other, n_head, n_set)) if total: print("kept %d / %d (%.1f%% reached Python, %.1f%% dropped in C)" % (admitted, total, 100.0 * admitted / total, 100.0 * dropped / total)) if __name__ == "__main__": try: main() except KeyboardInterrupt: pass pcapy-ng-2.1.0/examples/04_flow_heads.py000066400000000000000000000051231524067405300200450ustar00rootroot00000000000000#!/usr/bin/env python """Grab the START of every connection and ignore the bulk that follows. python 04_flow_heads.py capture.pcap sudo python 04_flow_heads.py eth0 flow_cutoff=N delivers the first N packets of each flow (TCP and UDP) as FLOW_HEAD; the rest is dropped in C. The opening packets are where the useful metadata is. As a concrete example we read the server name out of a TLS handshake (it travels in the clear, in the first packets of the connection) -- a thing BPF can't get you, because it needs flow state to know which packets are "the start". """ import os import sys import pcapy FLOW_HEAD = 1 ETH = 14 # IPv4 offset for Ethernet; see README for other link types def tls_server_name(packet, ip_off=ETH): """Return the SNI host from a TLS ClientHello at the start of a flow, or None. Defensive.""" try: b = bytearray(packet) ihl = (b[ip_off] & 0x0f) * 4 l4 = ip_off + ihl if b[ip_off + 9] != 6: # TCP only return None rec = l4 + (((b[l4 + 12] >> 4) & 0x0f) * 4) # TCP payload start if len(b) < rec + 43 or b[rec] != 0x16 or b[rec + 5] != 0x01: # TLS handshake/ClientHello return None p = rec + 5 + 4 + 2 + 32 p += 1 + b[p] # session id p += 2 + ((b[p] << 8) | b[p + 1]) # cipher suites p += 1 + b[p] # compression end = p + 2 + ((b[p] << 8) | b[p + 1]); p += 2 while p + 4 <= end and p + 4 <= len(b): etype = (b[p] << 8) | b[p + 1]; elen = (b[p + 2] << 8) | b[p + 3]; p += 4 if etype == 0x0000: # server_name extension np = p + 2 + 1 nlen = (b[np] << 8) | b[np + 1] return bytes(b[np + 2:np + 2 + nlen]).decode("ascii", "replace") p += elen except Exception: return None def open_source(arg): if arg and os.path.exists(arg): return pcapy.open_offline(arg) return pcapy.open_live(arg or pcapy.lookupdev(), 65535, True, 100) def main(): cap = open_source(sys.argv[1] if len(sys.argv) > 1 else None) seen = set() def on_head(hdr, data, cls): name = tls_server_name(bytes(data)) if name and name not in seen: seen.add(name) print("server name: %s" % name) cap.loop_filtered(-1, on_head, (1 << FLOW_HEAD), b"", 4) # first 4 pkts of each flow print("\n%d unique TLS server name(s) seen" % len(seen)) if __name__ == "__main__": try: main() except KeyboardInterrupt: pass pcapy-ng-2.1.0/examples/05_shared_memory_ring.py000066400000000000000000000051471524067405300216160ustar00rootroot00000000000000#!/usr/bin/env python """Worker-pool design: the capture thread classifies + admits in C and writes the kept packets straight into shared memory (GIL released the whole time); worker processes drain the buffer in parallel. No per-packet Python on the capture side. python 05_shared_memory_ring.py capture.pcap sudo python 05_shared_memory_ring.py eth0 Requires Python 3 (multiprocessing.shared_memory). Each record is [u32 caplen little-endian][u8 class][caplen bytes]. """ import os import sys from multiprocessing import shared_memory, Process import pcapy NAMES = ["OTHER", "FLOW_HEAD", "SET_MATCH"] RING_BYTES = 64 * 1024 * 1024 def consume(shm_name, start, end): shm = shared_memory.SharedMemory(name=shm_name) mv = memoryview(shm.buf) counts = {} off = start while off < end: ln = mv[off] | (mv[off + 1] << 8) | (mv[off + 2] << 16) | (mv[off + 3] << 24) cls = mv[off + 4] # packet bytes are mv[off+5 : off+5+ln]; here we just tally by class counts[cls] = counts.get(cls, 0) + 1 off += 5 + ln del mv shm.close() print(" worker %d-%d: %s" % (start, end, " ".join("%s=%d" % (NAMES[c], n) for c, n in sorted(counts.items())))) def slot_offsets(buf, written): offs = [] off = 0 mv = memoryview(buf) for _ in range(written): offs.append(off) ln = mv[off] | (mv[off + 1] << 8) | (mv[off + 2] << 16) | (mv[off + 3] << 24) off += 5 + ln offs.append(off) return offs def main(): arg = sys.argv[1] if len(sys.argv) > 1 else None cap = pcapy.open_offline(arg) if (arg and os.path.exists(arg)) \ else pcapy.open_live(arg or pcapy.lookupdev(), 65535, True, 100) shm = shared_memory.SharedMemory(create=True, size=RING_BYTES) try: # PRODUCER: keep flow heads + set matches; drop OTHER in C admit = (1 << 1) | (1 << 2) res = cap.loop_to_buffer(-1, shm.buf, admit, b"", 3) written, dropped, overflow, used = res[:4] print("producer: wrote %d packets (%.1f MB), dropped %d in C, overflow %d" % (written, used / 1e6, dropped, overflow)) # CONSUMERS: split the filled ring across 4 worker processes N = 4 offs = slot_offsets(shm.buf, written) procs = [] for k in range(N): a = offs[written * k // N] b = offs[written * (k + 1) // N] p = Process(target=consume, args=(shm.name, a, b)) p.start() procs.append(p) for p in procs: p.join() finally: shm.close() shm.unlink() if __name__ == "__main__": main() pcapy-ng-2.1.0/examples/06_live_stats.py000066400000000000000000000025661524067405300201210ustar00rootroot00000000000000#!/usr/bin/env python """Poll per-class counters from another thread WHILE the capture loop runs. sudo python 06_live_stats.py eth0 python 06_live_stats.py capture.pcap filtered_stats() is safe to read from a second thread; counters are monotonic so a torn read is harmless. Useful for a live stats line or to drive an adaptive controller. """ import os import sys import time import threading import pcapy def main(): arg = sys.argv[1] if len(sys.argv) > 1 else None cap = pcapy.open_offline(arg) if (arg and os.path.exists(arg)) \ else pcapy.open_live(arg or pcapy.lookupdev(), 65535, True, 100) def run(): # touch each admitted packet so the loop takes a moment (offline pcaps are fast) cap.loop_filtered(-1, lambda h, d, c: len(d), 7, b"", 4) t = threading.Thread(target=run) t.start() while t.is_alive(): adm, drop, other, head, sset = cap.filtered_stats() sys.stdout.write("\radmitted=%d dropped=%d other=%d flow_head=%d set_match=%d " % (adm, drop, other, head, sset)) sys.stdout.flush() time.sleep(0.2) t.join() final = cap.filtered_stats() print("\nfinal: admitted=%d dropped=%d (other,flow_head,set_match)=%s" % (final[0], final[1], final[2:])) if __name__ == "__main__": try: main() except KeyboardInterrupt: pass pcapy-ng-2.1.0/examples/07_fanout_scale.py000066400000000000000000000033341524067405300204020ustar00rootroot00000000000000#!/usr/bin/env python """Scale capture past one thread with PACKET_FANOUT (Linux). A single capture socket is one thread's worth of work. To use more cores on a fat pipe, open several sockets on the SAME interface, join them all to one kernel fanout group with set_fanout(), and read each in its own thread. The kernel hashes by flow, so every flow lands on exactly one socket -- the load is shared and no packet is captured twice. sudo python 07_fanout_scale.py eth0 4 # 4 capture threads on eth0 (Live capture needs root or CAP_NET_RAW. PACKET_FANOUT is Linux-only.) """ import sys import threading import pcapy SNAPLEN, PROMISC, TIMEOUT_MS = 65535, True, 100 def main(): dev = sys.argv[1] if len(sys.argv) > 1 else pcapy.lookupdev() n = int(sys.argv[2]) if len(sys.argv) > 2 else 4 group = 0x4711 # any 16-bit id; the same value ties the sockets together counts = [0] * n def worker(idx): cap = pcapy.open_live(dev, SNAPLEN, PROMISC, TIMEOUT_MS) cap.set_fanout(group, pcapy.PACKET_FANOUT_HASH) # join the group; flow-hashed distribution while True: try: hdr, _ = cap.next() except pcapy.PcapError: continue if hdr is not None: counts[idx] += 1 print("capturing on %s across %d fanout sockets ... (Ctrl-C to stop)" % (dev, n)) for i in range(n): t = threading.Thread(target=worker, args=(i,)) t.daemon = True t.start() try: while True: threading.Event().wait(2.0) print("per-socket packets: %s (total %d)" % (counts, sum(counts))) except KeyboardInterrupt: pass if __name__ == "__main__": main() pcapy-ng-2.1.0/examples/README.md000066400000000000000000000022661524067405300163410ustar00rootroot00000000000000# pcapy-ng examples Self-contained, runnable scripts. Each takes either a network device (live capture, needs privileges) or a `.pcap` file (no privileges) as its first argument. ```sh python 02_read_pcap.py capture.pcap # offline, no root sudo python 03_filter_in_c.py eth0 # live, needs root / CAP_NET_RAW ``` | Script | What it shows | |--------|---------------| | `01_sniff_live.py` | Classic capture: `open_live` + BPF filter + `next()`. | | `02_read_pcap.py` | Read a saved capture with `open_offline` and summarize it. | | `03_filter_in_c.py` | `loop_filtered`: tag + count packets, drop `OTHER` in C. | | `04_flow_heads.py` | Grab the first packets of each connection (`flow_cutoff`) — e.g. the TLS server name. | | `05_shared_memory_ring.py` | `loop_to_buffer` producer + multiprocess consumers (Python 3). | | `06_live_stats.py` | Poll `filtered_stats()` from another thread during capture. | | `07_fanout_scale.py` | `set_fanout`: spread one interface across several capture threads (Linux). | Don't have a capture handy? Make one (any traffic will do): ```sh sudo tcpdump -i any -s 2000 -w capture.pcap # Ctrl-C after a bit python 03_filter_in_c.py capture.pcap ``` pcapy-ng-2.1.0/pcap_pkthdr.cc000077500000000000000000000120321524067405300160450ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #define PY_SSIZE_T_CLEAN #include #include #include "pcapy.h" #include "pcap_pkthdr.h" #ifdef WIN32 #include #else #include #endif // internal pcapobject typedef struct { PyObject_HEAD struct timeval ts; bpf_u_int32 caplen; bpf_u_int32 len; } pkthdr; // Pkthdr_Type static void pcap_dealloc(pkthdr* pp) { PyObject_Del(pp); } // pcap methods static PyObject* p_getts(pkthdr* pp, PyObject* args); static PyObject* p_getcaplen(pkthdr* pp, PyObject* args); static PyObject* p_getlen(pkthdr* pp, PyObject* args); static PyMethodDef p_methods[] = { {"getts", (PyCFunction) p_getts, METH_VARARGS, "get timestamp tuple (seconds, microseconds) since the Epoch"}, {"getcaplen", (PyCFunction) p_getcaplen, METH_VARARGS, "returns the length of portion present"}, {"getlen", (PyCFunction) p_getlen, METH_VARARGS, "returns the length of the packet (off wire)"}, {NULL, NULL} /* sentinel */ }; static PyObject* pcap_getattr(pkthdr* pp, char* name) { #if PY_MAJOR_VERSION >= 3 PyObject *nameobj = PyUnicode_FromString(name); PyObject *attr = PyObject_GenericGetAttr((PyObject *)pp, nameobj); Py_DECREF(nameobj); return attr; #else return Py_FindMethod(p_methods, (PyObject*)pp, name); #endif } PyTypeObject Pkthdr_type = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(&PyType_Type, 0) "Pkthdr", /* tp_name */ sizeof(pkthdr), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)pcap_dealloc, /* tp_dealloc */ 0, /* tp_print */ (getattrfunc)pcap_getattr, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ NULL, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ p_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ #else PyObject_HEAD_INIT(NULL) 0, "Pkthdr", sizeof(pkthdr), 0, /* methods */ (destructor)pcap_dealloc, /* tp_dealloc*/ 0, /* tp_print*/ (getattrfunc)pcap_getattr, /* tp_getattr*/ 0, /* tp_setattr*/ 0, /* tp_compare*/ 0, /* tp_repr*/ 0, /* tp_as_number*/ 0, /* tp_as_sequence*/ 0, /* tp_as_mapping*/ #endif }; PyObject* new_pcap_pkthdr(const struct pcap_pkthdr* hdr) { if (PyType_Ready(&Pkthdr_type) < 0) return NULL; pkthdr *pp; pp = PyObject_New(pkthdr, &Pkthdr_type); if (pp == NULL) return NULL; pp->ts = hdr->ts; pp->caplen = hdr->caplen; pp->len = hdr->len; return (PyObject*)pp; } static PyObject* p_getts(pkthdr* pp, PyObject* args) { if (Py_TYPE(pp) != &Pkthdr_type) { PyErr_SetString(PcapError, "Not a pkthdr object"); return NULL; } return Py_BuildValue("(ll)", (long)pp->ts.tv_sec, (long)pp->ts.tv_usec); } static PyObject* p_getcaplen(pkthdr* pp, PyObject* args) { if (Py_TYPE(pp) != &Pkthdr_type) { PyErr_SetString(PcapError, "Not a pkthdr object"); return NULL; } return Py_BuildValue("l", pp->caplen); } static PyObject* p_getlen(pkthdr* pp, PyObject* args) { if (Py_TYPE(pp) != &Pkthdr_type) { PyErr_SetString(PcapError, "Not a pkthdr object"); return NULL; } return Py_BuildValue("l", pp->len); } int pkthdr_to_native(PyObject *pyhdr, struct pcap_pkthdr *hdr) { if (Py_TYPE(pyhdr) != &Pkthdr_type) { PyErr_SetString(PcapError, "Not a pkthdr object"); return -1; } pkthdr *pp = (pkthdr *) pyhdr; hdr->ts = pp->ts; hdr->caplen = pp->caplen; hdr->len = pp->len; return 0; } /* vim: set tabstop=2 shiftwidth=2 expandtab: */ pcapy-ng-2.1.0/pcap_pkthdr.h000077500000000000000000000007751524067405300157220ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #ifndef __pcap_pkthdr__ #define __pcap_pkthdr__ #include PyObject* new_pcap_pkthdr(const struct pcap_pkthdr* hdr); int pkthdr_to_native(PyObject *pyhdr, struct pcap_pkthdr *hdr); extern PyTypeObject Pkthdr_type; #endif // __pcap_pkthdr__ pcapy-ng-2.1.0/pcapdumper.cc000077500000000000000000000117701524067405300157160ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #define PY_SSIZE_T_CLEAN #include #include #include "pcapdumper.h" #include "pcap_pkthdr.h" #include "pcapy.h" // internal pcapdumper typedef struct { PyObject_HEAD pcap_dumper_t *dumper; } pcapdumper; static bool validate_pcapdumper(const pcapdumper* pp); // Pdumpertype static void pcap_dealloc(pcapdumper* pp) { if ( pp->dumper ) pcap_dump_close(pp->dumper); pp->dumper = NULL; PyObject_Del(pp); } // pcap methods static PyObject* p_close(pcapdumper* pp, PyObject* args); static PyObject* p_dump(pcapdumper* pp, PyObject* args); static PyMethodDef p_methods[] = { {"close", (PyCFunction) p_close, METH_VARARGS, "loops packet dispatching"}, {"dump", (PyCFunction) p_dump, METH_VARARGS, "dump a packet to the file"}, {NULL, NULL} /* sentinel */ }; static PyObject* pcap_getattr(pcapdumper* pp, char* name) { #if PY_MAJOR_VERSION >= 3 PyObject *nameobj = PyUnicode_FromString(name); PyObject *attr = PyObject_GenericGetAttr((PyObject *)pp, nameobj); Py_DECREF(nameobj); return attr; #else return Py_FindMethod(p_methods, (PyObject*)pp, name); #endif } PyTypeObject Pdumpertype = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(&PyType_Type, 0) "Dumper", /* tp_name */ sizeof(pcapdumper), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)pcap_dealloc, /* tp_dealloc */ 0, /* tp_print */ (getattrfunc)pcap_getattr, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ NULL, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ p_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ #else PyObject_HEAD_INIT(NULL) 0, "Dumper", sizeof(pcapdumper), 0, /* methods */ (destructor)pcap_dealloc, /* tp_dealloc */ 0, /* tp_print */ (getattrfunc)pcap_getattr, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_compare */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ #endif }; PyObject* new_pcapdumper(pcap_dumper_t *dumper) { if (PyType_Ready(&Pdumpertype) < 0) return NULL; pcapdumper *pp; pp = PyObject_New(pcapdumper, &Pdumpertype); if (pp == NULL) return NULL; pp->dumper = dumper; return (PyObject*)pp; } static PyObject* p_dump(pcapdumper* pp, PyObject* args) { PyObject *pyhdr; u_char *data; Py_ssize_t len; if(validate_pcapdumper(pp) == false){ return NULL; } #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTuple(args,"Oy#",&pyhdr,&data,&len)){ return NULL; } #else if (!PyArg_ParseTuple(args,"Os#",&pyhdr,&data,&len)){ return NULL; } #endif struct pcap_pkthdr hdr; if (-1 == pkthdr_to_native(pyhdr, &hdr)) return NULL; if (pp->dumper == NULL){ PyErr_SetString(PcapError, "Dumper is already closed."); return NULL; } pcap_dump((u_char *)pp->dumper, &hdr, data); Py_INCREF(Py_None); return Py_None; } // PdumperClose static PyObject* p_close(pcapdumper* pp, PyObject* args) { if(validate_pcapdumper(pp) == false){ return NULL; } if ( pp->dumper ) pcap_dump_close(pp->dumper); pp->dumper = NULL; Py_INCREF(Py_None); return Py_None; } static bool validate_pcapdumper(const pcapdumper* pp){ if (pp == NULL || Py_TYPE(pp) != &Pdumpertype) { PyErr_SetString(PcapError, "Not a pcapdumper object"); return false; } return true; } pcapy-ng-2.1.0/pcapdumper.h000077500000000000000000000006371524067405300155600ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #ifndef __pcapdumper__ #define __pcapdumper__ PyObject* new_pcapdumper(pcap_dumper_t *dumper); extern PyTypeObject Pdumpertype; #endif // __pcapdumper__ pcapy-ng-2.1.0/pcapobj.cc000077500000000000000000001516601524067405300151770ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #define PY_SSIZE_T_CLEAN #include #include #include "pcapobj.h" #include "pcapy.h" #include "pcapdumper.h" #include "pcap_pkthdr.h" #ifdef WIN32 #include #else #include #endif #if defined(__linux__) #include #include /* PACKET_FANOUT, PACKET_FANOUT_HASH, SOL_PACKET */ #endif // internal pcapobject typedef struct { PyObject_HEAD pcap_t *pcap; bpf_u_int32 net; bpf_u_int32 mask; /* F5: live per-class counters for loop_filtered, pollable mid-loop via filtered_stats(). [0]=admitted [1]=dropped [2..]=per-class-index counts (3 generic, up to 6 security) */ unsigned long long filt_counts[8]; int filt_profile; /* profile of the last loop_filtered run (0 generic, 1 security) */ } pcapobject; // PcapType static PyObject* p_close(pcapobject* pp, PyObject*) { if ( pp->pcap ) pcap_close(pp->pcap); pp->pcap = NULL; Py_RETURN_NONE; } static void pcap_dealloc(pcapobject* pp) { /* p_close() returns a new reference to None; dropping it on the floor leaked one reference to None per destroyed Reader (harmless once None became immortal in 3.12, but detected by tests/pcapytests.py testEOFValue on older interpreters). */ PyObject* res = p_close(pp, NULL); Py_XDECREF(res); PyObject_Del(pp); } static PyObject * err_closed(void) { PyErr_SetString(PyExc_ValueError, "pcap is closed"); return NULL; } // pcap methods static PyObject* p_getnet(pcapobject* pp, PyObject* args); static PyObject* p_getmask(pcapobject* pp, PyObject* args); static PyObject* p_setfilter( pcapobject* pp, PyObject* args ); static PyObject* p_next(pcapobject* pp, PyObject*); static PyObject* p_dispatch(pcapobject* pp, PyObject* args); static PyObject* p_loop(pcapobject* pp, PyObject* args); static PyObject* p_datalink(pcapobject* pp, PyObject* args); static PyObject* p_setdirection(pcapobject* pp, PyObject* args); static PyObject* p_setnonblock(pcapobject* pp, PyObject* args); static PyObject* p_getnonblock(pcapobject* pp, PyObject* args); static PyObject* p_dump_open(pcapobject* pp, PyObject* args); static PyObject* p_sendpacket(pcapobject* pp, PyObject* args); static PyObject* p_stats( pcapobject* pp, PyObject*); static PyObject* p__enter__( pcapobject* pp, PyObject*); static PyObject* p_getfd(pcapobject* pp, PyObject* args); static PyObject* p_set_fanout(pcapobject* pp, PyObject* args, PyObject* kwargs); static PyObject* p_set_snaplen(pcapobject* pp, PyObject* args); static PyObject* p_set_promisc(pcapobject* pp, PyObject* args); static PyObject* p_set_timeout(pcapobject* pp, PyObject* args); static PyObject* p_set_buffer_size(pcapobject* pp, PyObject* args); static PyObject* p_set_rfmon(pcapobject* pp, PyObject* args); static PyObject* p_activate(pcapobject* pp, PyObject* args); static PyObject* p_loop_filtered(pcapobject* pp, PyObject* args, PyObject* kwargs); static PyObject* p_filtered_stats(pcapobject* pp, PyObject*); static PyObject* p_loop_to_buffer(pcapobject* pp, PyObject* args, PyObject* kwargs); static PyObject* p_next_batch(pcapobject* pp, PyObject* args); static PyMethodDef p_methods[] = { {"loop", (PyCFunction) p_loop, METH_VARARGS, "loops packet dispatching"}, {"loop_filtered", (PyCFunction)(void(*)(void)) p_loop_filtered, METH_VARARGS | METH_KEYWORDS, "loop_filtered(cnt, cb, admit_mask=7, addr_set=b'', flow_cutoff=0, dpi_ports=None, tls_ports=None, dns_port=53, l2_offset=14, profile=0): classify each packet in C and invoke cb(hdr, data, cls) ONLY for admitted classes (admit_mask is a bitmask over class indices); dropped classes never build a Python object. GENERIC profile (default) classes: 0=OTHER, 1=FLOW_HEAD (within first flow_cutoff packets of a flow -- BPF can't track flows), 2=SET_MATCH (src/dst in addr_set, a packed 4-byte network-order IPv4 set of any size -- BPF can't hold large sets). l2_offset is where the IPv4 header begins (Ethernet=14, LINUX_SLL=16, DLT_RAW=0); VLAN tags are skipped. profile=1 selects a built-in security classifier (classes 100..105: DNS/HTTP/noise/IP-set/handshake/SYN) using dns_port/dpi_ports/tls_ports. Returns (admitted, dropped, )."}, {"filtered_stats", (PyCFunction) p_filtered_stats, METH_NOARGS, "filtered_stats(): live counter snapshot of the current/last loop_filtered run, safe to poll from another thread. Returns (admitted, dropped, ) matching the active profile."}, {"loop_to_buffer", (PyCFunction)(void(*)(void)) p_loop_to_buffer, METH_VARARGS | METH_KEYWORDS, "loop_to_buffer(cnt, writable_buf, admit_mask=7, addr_set=b'', flow_cutoff=0, dpi_ports=None, tls_ports=None, dns_port=53, l2_offset=14, profile=0): same classification as loop_filtered, but writes admitted packets into writable_buf (e.g. a multiprocessing.shared_memory buffer) as [u32 caplen LE][u8 class][caplen bytes] with the GIL released for the whole loop (no per-packet Python); worker processes drain it in parallel. Returns (written, dropped, overflow, bytes_used, )."}, {"next_batch", (PyCFunction) p_next_batch, METH_VARARGS, "next_batch(max_n): capture up to max_n packets in ONE call; returns (packet_bytes, packed_meta) where packed_meta is an array of 16-byte records, one per packet, each four native-endian uint32 (sec, usec, offset, caplen); offset/caplen slice the packet out of packet_bytes. Empty packed_meta = EOF/timeout. Amortizes the per-packet C<->Python boundary."}, {"dispatch", (PyCFunction) p_dispatch, METH_VARARGS, "dispatchs packets"}, {"next", (PyCFunction) p_next, METH_NOARGS, "returns next packet"}, {"setfilter", (PyCFunction) p_setfilter, METH_VARARGS, "compiles and sets a BPF capture filter"}, {"getnet", (PyCFunction) p_getnet, METH_VARARGS, "returns the network address for the device"}, {"getmask", (PyCFunction) p_getmask, METH_VARARGS, "returns the netmask for the device"}, {"datalink", (PyCFunction) p_datalink, METH_VARARGS, "returns the link layer type"}, {"getnonblock", (PyCFunction) p_getnonblock, METH_VARARGS, "returns the current `non-blocking' state"}, {"setnonblock", (PyCFunction) p_setnonblock, METH_VARARGS, "puts into `non-blocking' mode, or take it out, depending on the argument"}, {"setdirection", (PyCFunction) p_setdirection, METH_VARARGS, "set the direction for which packets will be captured"}, {"dump_open", (PyCFunction) p_dump_open, METH_VARARGS, "creates a dumper object"}, {"sendpacket", (PyCFunction) p_sendpacket, METH_VARARGS, "sends a packet through the interface"}, {"stats", (PyCFunction) p_stats, METH_NOARGS, "returns capture statistics"}, {"close", (PyCFunction) p_close, METH_NOARGS, "close the capture"}, {"set_snaplen", (PyCFunction)p_set_snaplen, METH_VARARGS, "set the snapshot length for a not-yet-activated capture handle"}, {"set_promisc", (PyCFunction)p_set_promisc, METH_VARARGS, "set promiscuous mode for a not-yet-activated capture handle"}, {"set_timeout", (PyCFunction)p_set_timeout, METH_VARARGS, "set the read timeout for a not-yet-activated capture handle"}, {"set_buffer_size", (PyCFunction)p_set_buffer_size, METH_VARARGS, "set the buffer size for a not-yet-activated capture handle"}, {"activate", (PyCFunction)p_activate, METH_NOARGS, "activate a capture handle created using create()"}, {"__enter__", (PyCFunction) p__enter__, METH_NOARGS, NULL}, {"__exit__", (PyCFunction) p_close, METH_VARARGS, NULL}, #ifndef WIN32 {"getfd", (PyCFunction) p_getfd, METH_VARARGS, "get selectable pcap fd"}, {"set_fanout", (PyCFunction)(void(*)(void)) p_set_fanout, METH_VARARGS | METH_KEYWORDS, "set_fanout(group_id, fanout_type=PACKET_FANOUT_HASH): Linux only. Join this (live, activated) capture's AF_PACKET socket to a PACKET_FANOUT group so the kernel load-balances one interface's packets across every capture handle sharing the same group_id. Open N handles, set_fanout(g) on each, and run N capture loops to scale capture past one thread (each flow stays on one handle with PACKET_FANOUT_HASH). Raises on non-Linux or a non-live handle."}, {"set_rfmon", (PyCFunction)p_set_rfmon, METH_VARARGS, "set monitor mode for a not-yet-activated capture handle"}, /* Available on Npcap, not on Winpcap. */ #endif {NULL, NULL} /* sentinel */ }; static PyObject* pcap_getattr(pcapobject* pp, char* name) { #if PY_MAJOR_VERSION >= 3 PyObject *nameobj = PyUnicode_FromString(name); PyObject *attr = PyObject_GenericGetAttr((PyObject *)pp, nameobj); Py_DECREF(nameobj); return attr; #else return Py_FindMethod(p_methods, (PyObject*)pp, name); #endif } PyTypeObject Pcaptype = { #if PY_MAJOR_VERSION >= 3 PyVarObject_HEAD_INIT(&PyType_Type, 0) "Reader", /* tp_name */ sizeof(pcapobject), /* tp_basicsize */ 0, /* tp_itemsize */ (destructor)pcap_dealloc, /* tp_dealloc */ 0, /* tp_print */ (getattrfunc)pcap_getattr, /* tp_getattr */ 0, /* tp_setattr */ 0, /* tp_reserved */ 0, /* tp_repr */ 0, /* tp_as_number */ 0, /* tp_as_sequence */ 0, /* tp_as_mapping */ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ NULL, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ p_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ #else PyObject_HEAD_INIT(NULL) 0, "Reader", sizeof(pcapobject), 0, /* methods */ (destructor)pcap_dealloc, /* tp_dealloc*/ 0, /* tp_print*/ (getattrfunc)pcap_getattr, /* tp_getattr*/ 0, /* tp_setattr*/ 0, /* tp_compare*/ 0, /* tp_repr*/ 0, /* tp_as_number*/ 0, /* tp_as_sequence*/ 0, /* tp_as_mapping*/ 0, /* tp_hash */ 0, /* tp_call */ 0, /* tp_str */ 0, /* tp_getattro */ 0, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT, /* tp_flags */ NULL, /* tp_doc */ 0, /* tp_traverse */ 0, /* tp_clear */ 0, /* tp_richcompare */ 0, /* tp_weaklistoffset */ 0, /* tp_iter */ 0, /* tp_iternext */ p_methods, /* tp_methods */ 0, /* tp_members */ 0, /* tp_getset */ 0, /* tp_base */ 0, /* tp_dict */ 0, /* tp_descr_get */ 0, /* tp_descr_set */ 0, /* tp_dictoffset */ 0, /* tp_init */ 0, /* tp_alloc */ 0, /* tp_new */ #endif }; PyObject* new_pcapobject(pcap_t *pcap, bpf_u_int32 net, bpf_u_int32 mask) { if (PyType_Ready(&Pcaptype) < 0) return NULL; pcapobject *pp; pp = PyObject_New(pcapobject, &Pcaptype); if (pp == NULL) return NULL; pp->pcap = pcap; pp->net = net; pp->mask = mask; return (PyObject*)pp; } static void ntos(char* dst, unsigned int n, int ip) { ip = htonl(ip); snprintf(dst, n, "%i.%i.%i.%i", ((ip >> 24) & 0xFF), ((ip >> 16) & 0xFF), ((ip >> 8) & 0xFF), (ip & 0xFF)); } static PyObject* p_getnet(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); char ip_str[20]; ntos(ip_str, sizeof(ip_str), pp->net); return Py_BuildValue("s", ip_str); } static PyObject* p_getmask(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); char ip_str[20]; ntos(ip_str, sizeof(ip_str), pp->mask); return Py_BuildValue("s", ip_str); } static PyObject* p_setfilter(pcapobject* pp, PyObject* args) { struct bpf_program bpfprog; int status; char* str; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); if (!PyArg_ParseTuple(args,"s:setfilter",&str)) return NULL; status = pcap_compile(pp->pcap, &bpfprog, str, 1, pp->mask); if (status) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } status = pcap_setfilter(pp->pcap, &bpfprog); if (status) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } Py_INCREF(Py_None); return Py_None; } static PyObject* p_next(pcapobject* pp, PyObject*) { struct pcap_pkthdr *hdr = NULL; const unsigned char *buf = (const unsigned char*)""; int err_code = 1; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); // allow threads as this might block Py_BEGIN_ALLOW_THREADS; err_code = pcap_next_ex(pp->pcap, &hdr, &buf); Py_END_ALLOW_THREADS; if(err_code == -1) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } PyObject *pkthdr; int _caplen = 0; if (err_code == 1) { pkthdr = new_pcap_pkthdr(hdr); _caplen = hdr->caplen; } else { pkthdr = Py_None; Py_INCREF(pkthdr); _caplen = 0; } if (pkthdr) { PyObject *ret = NULL; #if PY_MAJOR_VERSION >= 3 /* return bytes */ ret = Py_BuildValue("(Oy#)", pkthdr, buf, _caplen); #else ret = Py_BuildValue("(Os#)", pkthdr, buf, _caplen); #endif Py_DECREF(pkthdr); return ret; } PyErr_SetString(PcapError, "Can't build pkthdr"); return NULL; } struct PcapCallbackContext { PcapCallbackContext(pcap_t* p, PyObject* f, PyThreadState* ts) : ppcap_t(p), pyfunc(f), thread_state(ts) { Py_INCREF(pyfunc); } ~PcapCallbackContext() { Py_DECREF(pyfunc); } pcap_t* ppcap_t; PyObject *pyfunc; PyThreadState *thread_state; }; static void PythonCallBack(u_char *user, const struct pcap_pkthdr *header, const u_char *packetdata) { PyObject *arglist, *result; unsigned int *len; PcapCallbackContext *pctx; len = (unsigned int *)&header->caplen; pctx = (PcapCallbackContext *)user; PyEval_RestoreThread(pctx->thread_state); PyObject *hdr = new_pcap_pkthdr(header); #if PY_MAJOR_VERSION >= 3 /* pass bytes */ arglist = Py_BuildValue("Oy#", hdr, packetdata, *len); #else arglist = Py_BuildValue("Os#", hdr, packetdata, *len); #endif result = PyObject_CallObject(pctx->pyfunc,arglist); Py_XDECREF(arglist); if (result) Py_DECREF(result); Py_DECREF(hdr); if (!result) pcap_breakloop(pctx->ppcap_t); PyEval_SaveThread(); } static PyObject* p_dispatch(pcapobject* pp, PyObject* args) { int cant, ret; PyObject *PyFunc; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); if(!PyArg_ParseTuple(args,"iO:dispatch",&cant,&PyFunc)) return NULL; PcapCallbackContext ctx(pp->pcap, PyFunc, PyThreadState_Get()); PyEval_SaveThread(); ret = pcap_dispatch(pp->pcap, cant, PythonCallBack, (u_char*)&ctx); PyEval_RestoreThread(ctx.thread_state); // Fix: Check if a Python exception occurred inside the callback if (PyErr_Occurred()) { return NULL; } if(ret<0) { if (ret!=-2) /* pcap error, pcap_breakloop was not called so error is not set */ PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; // This handles the error case correctly now } return Py_BuildValue("i", ret); } static PyObject* p_stats(pcapobject* pp, PyObject*) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); struct pcap_stat stats; if (-1 == pcap_stats(pp->pcap, &stats)) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } return Py_BuildValue("III", stats.ps_recv, stats.ps_drop, stats.ps_ifdrop); } static PyObject* p__enter__( pcapobject* pp, PyObject*) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); Py_INCREF(pp); return (PyObject*)pp; } static PyObject* p_dump_open(pcapobject* pp, PyObject* args) { char *filename; pcap_dumper_t *ret; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); if(!PyArg_ParseTuple(args,"s",&filename)) return NULL; ret = pcap_dump_open(pp->pcap, filename); if (ret==NULL) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } return new_pcapdumper(ret); } static PyObject* p_loop(pcapobject* pp, PyObject* args) { int cant, ret; PyObject *PyFunc; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); if(!PyArg_ParseTuple(args,"iO:loop",&cant,&PyFunc)) return NULL; PcapCallbackContext ctx(pp->pcap, PyFunc, PyThreadState_Get()); PyEval_SaveThread(); ret = pcap_loop(pp->pcap, cant, PythonCallBack, (u_char*)&ctx); PyEval_RestoreThread(ctx.thread_state); if(ret<0) { if (ret!=-2) /* pcap error, pcap_breakloop was not called so error is not set */ PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } Py_INCREF(Py_None); return Py_None; } static PyObject* p_datalink(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int type = pcap_datalink(pp->pcap); return Py_BuildValue("i", type); } static PyObject* p_setdirection(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); pcap_direction_t direction; if (!PyArg_ParseTuple(args, "i", &direction)) return NULL; int ret = pcap_setdirection(pp->pcap, direction); if (-1 == ret) { PyErr_SetString(PcapError, "Failed setting direction"); return NULL; } Py_INCREF(Py_None); return Py_None; } static PyObject* p_setnonblock(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int state; if (!PyArg_ParseTuple(args, "i", &state)) return NULL; char errbuf[PCAP_ERRBUF_SIZE]; int ret = pcap_setnonblock(pp->pcap, state, errbuf); if (-1 == ret) { PyErr_SetString(PcapError, errbuf); return NULL; } Py_INCREF(Py_None); return Py_None; } static PyObject* p_getnonblock(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); char errbuf[PCAP_ERRBUF_SIZE]; int state = pcap_getnonblock(pp->pcap, errbuf); if (-1 == state) { PyErr_SetString(PcapError, errbuf); return NULL; } return Py_BuildValue("i", state); } static PyObject* p_set_snaplen(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int snaplen; if (!PyArg_ParseTuple(args, "i", &snaplen)) return NULL; int ret = pcap_set_snaplen(pp->pcap, snaplen); return Py_BuildValue("i", ret); } static PyObject* p_set_promisc(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int promisc; if (!PyArg_ParseTuple(args, "i", &promisc)) return NULL; int ret = pcap_set_promisc(pp->pcap, promisc); return Py_BuildValue("i", ret); } static PyObject* p_set_timeout(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int to_ms; if (!PyArg_ParseTuple(args, "i", &to_ms)) return NULL; int ret = pcap_set_timeout(pp->pcap, to_ms); return Py_BuildValue("i", ret); } static PyObject* p_set_buffer_size(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int buffer_size; if (!PyArg_ParseTuple(args, "i", &buffer_size)) return NULL; int ret = pcap_set_buffer_size(pp->pcap, buffer_size); return Py_BuildValue("i", ret); } static PyObject* p_set_rfmon(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int rfmon; if (!PyArg_ParseTuple(args, "i", &rfmon)) return NULL; int ret = pcap_set_rfmon(pp->pcap, rfmon); return Py_BuildValue("i", ret); } static PyObject* p_activate(pcapobject* pp, PyObject*) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int ret = pcap_activate(pp->pcap); return Py_BuildValue("i", ret); } static PyObject* p_sendpacket(pcapobject* pp, PyObject* args) { int status; unsigned char* str; Py_ssize_t length; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); #if PY_MAJOR_VERSION >= 3 /* accept bytes */ if (!PyArg_ParseTuple(args,"y#", &str, &length)) { return NULL; } #else if (!PyArg_ParseTuple(args,"s#", &str, &length)) { return NULL; } #endif status = pcap_sendpacket(pp->pcap, str, length); if (status) { PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } Py_INCREF(Py_None); return Py_None; } #ifndef WIN32 static PyObject* p_getfd(pcapobject* pp, PyObject* args) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); int fd = pcap_get_selectable_fd(pp->pcap); return Py_BuildValue("i", fd); } #endif /* set_fanout(group_id, fanout_type=PACKET_FANOUT_HASH): join this live AF_PACKET capture to a kernel PACKET_FANOUT group so one interface's traffic is load-balanced across every handle in the same group. Linux-only; needs a live, activated handle (pcap_fileno gives the socket fd). */ static PyObject* p_set_fanout(pcapobject* pp, PyObject* args, PyObject* kwargs) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); #if defined(__linux__) && defined(PACKET_FANOUT) int group_id; int fanout_type = PACKET_FANOUT_HASH; static char* kwlist[] = { (char*)"group_id", (char*)"fanout_type", NULL }; if (!PyArg_ParseTupleAndKeywords(args, kwargs, "i|i:set_fanout", kwlist, &group_id, &fanout_type)) return NULL; int fd = pcap_fileno(pp->pcap); if (fd < 0) { PyErr_SetString(PcapError, "no socket fd for this capture (PACKET_FANOUT needs a live AF_PACKET capture, not an offline pcap)"); return NULL; } /* PACKET_FANOUT_FLAG_DEFRAG is 0x8000, so `fanout_type << 16` on a signed int would shift into the sign bit (UB); build the word in unsigned and convert once. */ int arg = (int)(((unsigned)group_id & 0xffffu) | ((unsigned)fanout_type << 16)); if (setsockopt(fd, SOL_PACKET, PACKET_FANOUT, &arg, sizeof(arg)) < 0) { PyErr_SetFromErrno(PcapError); return NULL; } Py_INCREF(Py_None); return Py_None; #else (void) args; (void) kwargs; PyErr_SetString(PcapError, "PACKET_FANOUT is only available on Linux"); return NULL; #endif } /* ---- pcapy-ng FAST: in-C severity classify + admit (generic prefilter) ---- */ /* hand-coded fast classifier: 0=DNS (port dns_port), 1=DPI (PSH on dpi_ports), 2=noise. Ports are configurable per loop_filtered() call so the consumer can match its own capture filter (Maltrail's CAPTURE_FILTER) instead of a hardcoded set. */ static const unsigned short MT_DPI_PORTS_DEFAULT[] = {80, 8080, 3128, 8000, 8118, 1080}; static const unsigned short MT_TLS_PORTS_DEFAULT[] = {443}; struct mt_classcfg { unsigned short dns_port; /* default 53 */ const unsigned short* dpi_ports; /* TCP PSH on any of these -> DPI (class 1) */ unsigned int ndpi; const unsigned short* tls_ports; /* flow-cutoff head capture on these -> HEAD (class 4) */ unsigned int ntls; unsigned int l2_off; /* byte offset where the IPv4 header begins: Ethernet=14, LINUX_SLL=16, LINUX_SLL2=20, DLT_RAW/null=0 */ int profile; /* 0 = generic (OTHER/FLOW_HEAD/SET_MATCH); 1 = security */ }; /* Single-pass classify: returns the class AND emits the parsed L3/L4 offsets so the caller's noise-refinement (IOC set + flow-cutoff) reuses them instead of re-parsing. `cand` is set when the packet is IPv4 TCP/UDP with valid ports (the only case the refinement cares about). Noise is ~94% of a big pipe, so dropping the second parse there is the hot-path win. */ /* classes: 0=DNS 1=DPI 2=noise 3=IOC-IP 4=HEAD(TLS/QUIC handshake) 5=SYN(TCP connect, heuristics). cand = IPv4 TCP/UDP with valid ports; ipv4 = any IPv4 packet (l3 valid) -> IOC works for ICMP too. */ struct mt_pktinfo { int cand; int ipv4; int proto; unsigned int l3; unsigned int l4; unsigned short sport, dport; }; static inline int mt_classify(const struct mt_classcfg* cfg, const unsigned char* p, unsigned int len, struct mt_pktinfo* o) { o->cand = 0; o->ipv4 = 0; unsigned int l2 = cfg->l2_off; /* Find the IPv4 header, transparently skipping up to two 802.1Q/802.1ad VLAN tags (TPID 0x8100 / 0x88a8 in the two bytes preceding the current offset). Validate IPv4 by the version nibble, which is datalink-agnostic (Ethernet / SLL / raw). */ int vguard = 0; for (;;) { if (len < l2 + 20) return 2; /* need a full minimal IPv4 header */ if ((p[l2] >> 4) == 4) break; /* IPv4 header located */ if (vguard++ >= 2 || l2 < 2) return 2; if ((p[l2 - 2] == 0x81 && p[l2 - 1] == 0x00) || (p[l2 - 2] == 0x88 && p[l2 - 1] == 0xa8)) l2 += 4; /* skip one VLAN tag */ else return 2; /* not IPv4 and not a VLAN tag */ } o->l3 = l2; o->ipv4 = 1; /* IPv4 located -> IOC works for any proto */ unsigned int ihl = (p[l2] & 0x0f) * 4; if (ihl < 20) return 2; /* invalid IHL */ unsigned int l4 = l2 + ihl; o->proto = p[l2 + 9]; if (o->proto != 17 && o->proto != 6) return 2; /* ICMP/other: noise, but ipv4 set for IOC */ if (len < l4 + 4) return 2; int proto = o->proto; unsigned short sport = (p[l4] << 8) | p[l4 + 1]; unsigned short dport = (p[l4 + 2] << 8) | p[l4 + 3]; o->cand = 1; o->l4 = l4; o->sport = sport; o->dport = dport; if (sport == cfg->dns_port || dport == cfg->dns_port) return 0; /* DNS */ if (proto == 6 && len >= l4 + 14) { unsigned char flags = p[l4 + 13]; if (flags == 0x02) return 5; /* pure SYN -> connection (scan/infection heuristics) */ /* DPI: a payload-bearing TCP packet on a watched (HTTP) port. NOT gated on PSH, because Maltrail inspects HTTP requests AND responses (any segment carrying "HTTP/"), and a header-bearing segment is not always PSH. Bare ACKs (no payload) carry no HTTP and, if to/from a trail IP, are caught by the IOC class instead. */ unsigned int tcphl = ((p[l4 + 12] >> 4) & 0x0f) * 4; if (tcphl >= 20 && len > l4 + tcphl) { /* has TCP payload */ for (unsigned i = 0; i < cfg->ndpi; i++) if (sport == cfg->dpi_ports[i] || dport == cfg->dpi_ports[i]) return 1; /* DPI */ } } return 2; /* noise */ } static inline int mt_is_tls_port(const struct mt_classcfg* cfg, unsigned short sport, unsigned short dport) { for (unsigned i = 0; i < cfg->ntls; i++) if (sport == cfg->tls_ports[i] || dport == cfg->tls_ports[i]) return 1; return 0; } #include #include /* tiny open-addressing uint32 IP set (network-order keys; 0.0.0.0 reserved as empty sentinel) */ struct mt_ipset { unsigned int* slots; unsigned int shift; }; /* Slot index for an address: Fibonacci hashing -- multiply, then take the HIGH bits. This used to be `(ip * 2654435761u) & mask`, which is the classic way to get a multiplicative hash wrong: the low bits of a product depend only on the low bits of the factors, so masking throws away exactly the bits the multiply mixed. The key here is the raw network-order word, so on a little-endian host the last octet of the address lands in the TOP byte -- the part that never reached the index. Expanding a CIDR block therefore mapped a whole /24 onto a single slot and linear probing degenerated into a linear scan (measured: ~6850 probes per lookup on a 100k-entry sequential set, versus ~1.3 on random addresses; the fix brings sequential sets to ~1.0). Taking the top `bits` of the product makes the index depend on all 32 input bits for the same work as before: one multiply and a shift instead of an AND. `shift` is 32 - bits, and since the table is a power of two the probe mask is just (0xffffffff >> shift), so the struct still carries a single scalar and lookups do a single load. */ static inline unsigned int ipset_slot(unsigned int shift, unsigned int ip) { return (unsigned int)((ip * 2654435761u) >> shift); } static struct mt_ipset* ipset_build(const unsigned char* data, Py_ssize_t len) { unsigned int n = (unsigned int)(len / 4), cap = 16, bits = 4; while (cap < n * 2 + 1 && bits < 31) { cap <<= 1; bits++; } struct mt_ipset* s = (struct mt_ipset*)malloc(sizeof(*s)); if (!s) return NULL; s->slots = (unsigned int*)calloc(cap, sizeof(unsigned int)); if (!s->slots) { free(s); return NULL; } s->shift = 32 - bits; unsigned int mask = cap - 1; for (unsigned int i = 0; i < n; i++) { unsigned int ip; memcpy(&ip, data + i * 4, 4); if (!ip) continue; unsigned int j = ipset_slot(s->shift, ip); while (s->slots[j] && s->slots[j] != ip) j = (j + 1) & mask; s->slots[j] = ip; } return s; } static inline int ipset_has(struct mt_ipset* s, unsigned int ip) { if (!s || !ip) return 0; unsigned int shift = s->shift, mask = 0xffffffffu >> shift; unsigned int j = ipset_slot(shift, ip); while (s->slots[j]) { if (s->slots[j] == ip) return 1; j = (j + 1) & mask; } return 0; } static void ipset_free(struct mt_ipset* s) { if (s) { free(s->slots); free(s); } } /* approximate per-flow packet counter (overwrite-on-collision LRU); for flow-cutoff */ struct mt_flowtab { unsigned long long* keys; unsigned int* cnt; unsigned int mask; }; static struct mt_flowtab* flowtab_build(unsigned int bits) { unsigned int cap = 1u << bits; struct mt_flowtab* f = (struct mt_flowtab*)malloc(sizeof(*f)); if (!f) return NULL; f->keys = (unsigned long long*)calloc(cap, sizeof(unsigned long long)); f->cnt = (unsigned int*)calloc(cap, sizeof(unsigned int)); if (!f->keys || !f->cnt) { free(f->keys); free(f->cnt); free(f); return NULL; } f->mask = cap - 1; return f; } /* 64-bit key over the directional 5-tuple (>96 bits in, 64 out, so folding is unavoidable). The fold must avalanche: a SLOT collision is benign by design (the full-key compare below resets the counter, so the flow simply restarts and admits an extra head), but a KEY collision makes two flows share one counter and can suppress a real FLOW_HEAD -- the one failure mode this design rules out. A splitmix64 finalizer spreads every input bit over all 64 output bits, putting that at ~2^-64 per flow pair; an XOR of small multipliers would leave the destination address confined to the low ~48 bits. Ports are widened before shifting, so nothing is shifted into the sign bit of an int. */ static inline unsigned long long flow_key(unsigned int sip, unsigned int dip, unsigned int sport, unsigned int dport, unsigned int proto) { unsigned long long a = ((unsigned long long)sip << 32) | dip; unsigned long long b = ((unsigned long long)(sport & 0xffffu) << 48) | ((unsigned long long)(dport & 0xffffu) << 32) | (proto & 0xffu); a ^= b * 0x9E3779B97F4A7C15ULL; a ^= a >> 30; a *= 0xBF58476D1CE4E5B9ULL; a ^= a >> 27; a *= 0x94D049BB133111EBULL; a ^= a >> 31; return a; } static inline unsigned int flow_seen(struct mt_flowtab* f, unsigned long long key) { unsigned int slot = (unsigned int)key & f->mask; /* key is already fully avalanched */ if (f->keys[slot] != key) { f->keys[slot] = key; f->cnt[slot] = 0; } /* new flow / evicted */ return f->cnt[slot]++; /* count before increment */ } static void flowtab_free(struct mt_flowtab* f) { if (f) { free(f->keys); free(f->cnt); free(f); } } /* Locate the IPv4 header, skipping up to two 802.1Q/802.1ad VLAN tags; return its offset or -1. */ static inline int locate_ipv4(unsigned int l2, const unsigned char* p, unsigned int len) { int vguard = 0; for (;;) { if (len < l2 + 20) return -1; if ((p[l2] >> 4) == 4) return (int)l2; if (vguard++ >= 2 || l2 < 2) return -1; if ((p[l2 - 2] == 0x81 && p[l2 - 1] == 0x00) || (p[l2 - 2] == 0x88 && p[l2 - 1] == 0xa8)) l2 += 4; else return -1; } } /* GENERIC profile (the default). Three classes, all of which are things a stateless BPF filter cannot express, so the method earns its place next to setfilter(): 0 OTHER didn't match anything below 1 FLOW_HEAD within the first `flow_cutoff` packets of its flow (TCP or UDP) -- the start of a connection, where the interesting bytes live (e.g. the TLS SNI). BPF has no flow state. 2 SET_MATCH src or dst address is in the caller-supplied address set (any size, any protocol). */ static inline int classify_generic(const struct mt_classcfg* cfg, struct mt_ipset* set, struct mt_flowtab* flowtab, unsigned int flow_cutoff, const unsigned char* p, unsigned int len) { int l3 = locate_ipv4(cfg->l2_off, p, len); if (l3 < 0) return 0; /* OTHER (not IPv4) */ unsigned int sip, dip; memcpy(&sip, p + l3 + 12, 4); memcpy(&dip, p + l3 + 16, 4); if (set && (ipset_has(set, sip) || ipset_has(set, dip))) return 2; /* SET_MATCH */ if (flowtab && flow_cutoff) { int proto = p[l3 + 9]; if (proto == 6 || proto == 17) { /* flow key needs ports */ unsigned int ihl = (p[l3] & 0x0f) * 4, l4 = l3 + ihl; if (ihl >= 20 && len >= l4 + 4) { unsigned int sp = ((unsigned)p[l4] << 8) | p[l4 + 1]; unsigned int dp = ((unsigned)p[l4 + 2] << 8) | p[l4 + 3]; if (flow_seen(flowtab, flow_key(sip, dip, sp, dp, (unsigned)proto)) < flow_cutoff) return 1; /* FLOW_HEAD */ } } } return 0; /* OTHER */ } /* shared classify + noise-refinement (IOC set + flow-cutoff). Returns final class 0..4. */ static inline int classify_refine(const struct mt_classcfg* cfg, struct mt_ipset* ioc, struct mt_flowtab* flowtab, unsigned int flow_cutoff, const unsigned char* pkt, unsigned int caplen) { struct mt_pktinfo pi; int cls = mt_classify(cfg, pkt, caplen, &pi); if (cls == 2 && pi.ipv4 && (ioc || flowtab)) { unsigned int l2 = pi.l3; /* VLAN-resolved IPv4 header offset */ unsigned int sip, dip; memcpy(&sip, pkt + l2 + 12, 4); memcpy(&dip, pkt + l2 + 16, 4); if (ioc && (ipset_has(ioc, sip) || ipset_has(ioc, dip))) cls = 3; /* known-bad IP, ANY protocol incl ICMP */ else if (pi.cand && flowtab && mt_is_tls_port(cfg, pi.sport, pi.dport)) { unsigned long long key = flow_key(sip, dip, pi.sport, pi.dport, (unsigned)pi.proto); if (flow_seen(flowtab, key) < flow_cutoff) cls = 4; } } return cls; } /* Pick the classifier for the active profile. Returns a dense class INDEX (0-based) used for the admit mask and counters; the public class id delivered to the callback is profile_base()+index. */ static inline int classify_dispatch(const struct mt_classcfg* cfg, struct mt_ipset* set, struct mt_flowtab* flowtab, unsigned int flow_cutoff, const unsigned char* pkt, unsigned int caplen) { if (cfg->profile == 1) return classify_refine(cfg, set, flowtab, flow_cutoff, pkt, caplen); /* security: 0..5 */ return classify_generic(cfg, set, flowtab, flow_cutoff, pkt, caplen); /* generic: 0..2 */ } /* security-profile class ids are offset by 100 so a general user's 0/1/2 never collide with them */ #define MT_PROFILE_BASE 100 static inline int profile_base(const struct mt_classcfg* cfg) { return cfg->profile == 1 ? MT_PROFILE_BASE : 0; } struct FilteredContext { pcap_t* ppcap_t; PyObject* pyfunc; PyThreadState* thread_state; int admit_mask; struct mt_ipset* ioc; struct mt_flowtab* flowtab; unsigned int flow_cutoff; struct mt_classcfg cfg; unsigned long long* live; /* -> pcapobject.filt_counts: [0]adm [1]drop [2..7]=class index 0..5 */ }; static void FilteredCallBack(u_char* user, const struct pcap_pkthdr* header, const u_char* packetdata) { FilteredContext* c = (FilteredContext*)user; int idx = classify_dispatch(&c->cfg, c->ioc, c->flowtab, c->flow_cutoff, packetdata, header->caplen); c->live[2 + idx]++; if (!((c->admit_mask >> idx) & 1)) { c->live[1]++; return; } /* dropped in C, no GIL, no Python */ c->live[0]++; int cls = profile_base(&c->cfg) + idx; /* public class id for the callback */ Py_ssize_t caplen = header->caplen; PyEval_RestoreThread(c->thread_state); PyObject* hdr = new_pcap_pkthdr(header); #if PY_MAJOR_VERSION >= 3 PyObject* arglist = Py_BuildValue("Oy#i", hdr, packetdata, caplen, cls); #else PyObject* arglist = Py_BuildValue("Os#i", hdr, packetdata, caplen, cls); #endif PyObject* result = PyObject_CallObject(c->pyfunc, arglist); Py_XDECREF(arglist); if (result) Py_DECREF(result); Py_DECREF(hdr); if (!result) pcap_breakloop(c->ppcap_t); c->thread_state = PyEval_SaveThread(); } /* Convert a Python sequence of ints into a malloc'd uint16 port array. Returns the array (caller frees) and sets *outn; on None/empty returns NULL with *outn=0. On a type error returns NULL and sets *err=1. */ static unsigned short* seq_to_ports(PyObject* seq, unsigned int* outn, int* err) { *outn = 0; *err = 0; if (!seq || seq == Py_None) return NULL; PyObject* fast = PySequence_Fast(seq, "ports must be a sequence of ints"); if (!fast) { *err = 1; return NULL; } Py_ssize_t n = PySequence_Fast_GET_SIZE(fast); if (n <= 0) { Py_DECREF(fast); return NULL; } unsigned short* ports = (unsigned short*)malloc((size_t)n * sizeof(unsigned short)); if (!ports) { Py_DECREF(fast); PyErr_NoMemory(); *err = 1; return NULL; } for (Py_ssize_t i = 0; i < n; i++) { long v = PyLong_AsLong(PySequence_Fast_GET_ITEM(fast, i)); if (v == -1 && PyErr_Occurred()) { free(ports); Py_DECREF(fast); *err = 1; return NULL; } ports[i] = (unsigned short)(v & 0xffff); } Py_DECREF(fast); *outn = (unsigned int)n; return ports; } static PyObject* p_loop_filtered(pcapobject* pp, PyObject* args, PyObject* kwargs) { static char* kwlist[] = { (char*)"cnt", (char*)"cb", (char*)"admit_mask", (char*)"addr_set", (char*)"flow_cutoff", (char*)"dpi_ports", (char*)"tls_ports", (char*)"dns_port", (char*)"l2_offset", (char*)"profile", NULL }; int cant, admit_mask = 7; /* generic default: admit OTHER(0)+FLOW_HEAD(1)+SET_MATCH(2) */ int flow_cutoff = 0; int dns_port = 53; int l2_offset = 14; /* Ethernet default; LINUX_SLL=16, DLT_RAW/null=0 */ int profile = 0; /* 0 = generic; 1 = security (DNS/HTTP/IP-set/handshake/SYN at 100+) */ PyObject* PyFunc; PyObject* dpi_seq = NULL; PyObject* tls_seq = NULL; const char* iocbuf = NULL; Py_ssize_t ioclen = 0; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "iO|iy#iOOiii:loop_filtered", kwlist, &cant, &PyFunc, &admit_mask, &iocbuf, &ioclen, &flow_cutoff, &dpi_seq, &tls_seq, &dns_port, &l2_offset, &profile)) return NULL; #else if (!PyArg_ParseTupleAndKeywords(args, kwargs, "iO|is#iOOiii:loop_filtered", kwlist, &cant, &PyFunc, &admit_mask, &iocbuf, &ioclen, &flow_cutoff, &dpi_seq, &tls_seq, &dns_port, &l2_offset, &profile)) return NULL; #endif if (l2_offset < 0 || l2_offset > 64) { PyErr_SetString(PyExc_ValueError, "l2_offset out of range [0,64]"); return NULL; } if (profile != 0 && profile != 1) { PyErr_SetString(PyExc_ValueError, "profile must be 0 (generic) or 1 (security)"); return NULL; } int perr = 0; unsigned int ndpi = 0, ntls = 0; unsigned short* dpi_ports = seq_to_ports(dpi_seq, &ndpi, &perr); if (perr) { free(dpi_ports); return NULL; } unsigned short* tls_ports = seq_to_ports(tls_seq, &ntls, &perr); if (perr) { free(dpi_ports); free(tls_ports); return NULL; } /* None/absent -> built-in defaults; an explicitly provided sequence (even empty) is honored */ int dpi_given = (dpi_seq && dpi_seq != Py_None); int tls_given = (tls_seq && tls_seq != Py_None); FilteredContext c; c.ppcap_t = pp->pcap; c.pyfunc = PyFunc; c.admit_mask = admit_mask; c.cfg.dns_port = (unsigned short)dns_port; c.cfg.dpi_ports = dpi_given ? dpi_ports : MT_DPI_PORTS_DEFAULT; c.cfg.ndpi = dpi_given ? ndpi : (unsigned)(sizeof(MT_DPI_PORTS_DEFAULT)/sizeof(MT_DPI_PORTS_DEFAULT[0])); c.cfg.tls_ports = tls_given ? tls_ports : MT_TLS_PORTS_DEFAULT; c.cfg.ntls = tls_given ? ntls : (unsigned)(sizeof(MT_TLS_PORTS_DEFAULT)/sizeof(MT_TLS_PORTS_DEFAULT[0])); c.cfg.l2_off = (unsigned)l2_offset; c.cfg.profile = profile; c.ioc = (iocbuf && ioclen >= 4) ? ipset_build((const unsigned char*)iocbuf, ioclen) : NULL; c.flow_cutoff = (flow_cutoff > 0) ? (unsigned)flow_cutoff : 0; c.flowtab = c.flow_cutoff ? flowtab_build(20) : NULL; /* 1M slots ~12MB */ c.live = pp->filt_counts; for (int i = 0; i < 8; i++) pp->filt_counts[i] = 0; /* reset for this run; pollable mid-loop */ pp->filt_profile = profile; c.thread_state = PyEval_SaveThread(); int ret = pcap_loop(pp->pcap, cant, FilteredCallBack, (u_char*)&c); PyEval_RestoreThread(c.thread_state); ipset_free(c.ioc); flowtab_free(c.flowtab); free(dpi_ports); free(tls_ports); if (PyErr_Occurred()) return NULL; if (ret < 0) { if (ret != -2) PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } unsigned long long* s = pp->filt_counts; /* generic -> (admitted, dropped, other, flow_head, set_match); security -> (admitted, dropped, dns, http, noise, ipset, handshake, syn) */ if (profile == 1) return Py_BuildValue("(KKKKKKKK)", s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]); return Py_BuildValue("(KKKKK)", s[0], s[1], s[2], s[3], s[4]); } /* F5: read the live per-class counters of an in-flight (or finished) loop_filtered. Safe to call from another Python thread while loop_filtered runs (counters are monotonic; a torn read is harmless for stats). Returns (admitted,dropped,dns,dpi,noise,ioc,head). */ static PyObject* p_filtered_stats(pcapobject* pp, PyObject*) { if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } unsigned long long* s = pp->filt_counts; if (pp->filt_profile == 1) return Py_BuildValue("(KKKKKKKK)", s[0], s[1], s[2], s[3], s[4], s[5], s[6], s[7]); return Py_BuildValue("(KKKKK)", s[0], s[1], s[2], s[3], s[4]); } /* ---- F2 (producer half): classify+admit and write admitted packets straight into a caller-provided writable buffer (the shared-memory ring's backing store), with ZERO per-packet Python crossing. Proves the producer can feed a ring at full C speed (GIL fully released for the whole loop); worker processes drain the ring in parallel. Slot layout per admitted packet: [u32 caplen-LE][u8 cls][caplen bytes]. Stops writing when the buffer is full (overflow counted) but keeps classifying so stats stay exact. */ struct BufferCtx { unsigned char* buf; size_t cap, used; int admit_mask; struct mt_ipset* ioc; struct mt_flowtab* flowtab; unsigned int flow_cutoff; struct mt_classcfg cfg; unsigned long long written, dropped, overflow, cls[6]; }; static void BufferCallBack(u_char* user, const struct pcap_pkthdr* h, const u_char* pkt) { struct BufferCtx* b = (struct BufferCtx*)user; int idx = classify_dispatch(&b->cfg, b->ioc, b->flowtab, b->flow_cutoff, pkt, h->caplen); b->cls[idx]++; if (!((b->admit_mask >> idx) & 1)) { b->dropped++; return; } unsigned int cl = h->caplen; size_t need = 5 + cl; if (b->used + need > b->cap) { b->overflow++; return; } /* ring backing full; consumer behind */ unsigned char* p = b->buf + b->used; p[0] = cl & 0xff; p[1] = (cl >> 8) & 0xff; p[2] = (cl >> 16) & 0xff; p[3] = (cl >> 24) & 0xff; p[4] = (unsigned char)(profile_base(&b->cfg) + idx); /* public class id in the ring */ memcpy(p + 5, pkt, cl); b->used += need; b->written++; } static PyObject* p_loop_to_buffer(pcapobject* pp, PyObject* args, PyObject* kwargs) { static char* kwlist[] = { (char*)"cnt", (char*)"writable_buf", (char*)"admit_mask", (char*)"addr_set", (char*)"flow_cutoff", (char*)"dpi_ports", (char*)"tls_ports", (char*)"dns_port", (char*)"l2_offset", (char*)"profile", NULL }; int cant, admit_mask = 7, flow_cutoff = 0, dns_port = 53, l2_offset = 14, profile = 0; Py_buffer view; PyObject* dpi_seq = NULL; PyObject* tls_seq = NULL; const char* iocbuf = NULL; Py_ssize_t ioclen = 0; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); #if PY_MAJOR_VERSION >= 3 if (!PyArg_ParseTupleAndKeywords(args, kwargs, "iw*|iy#iOOiii:loop_to_buffer", kwlist, &cant, &view, &admit_mask, &iocbuf, &ioclen, &flow_cutoff, &dpi_seq, &tls_seq, &dns_port, &l2_offset, &profile)) return NULL; #else if (!PyArg_ParseTupleAndKeywords(args, kwargs, "iw*|is#iOOiii:loop_to_buffer", kwlist, &cant, &view, &admit_mask, &iocbuf, &ioclen, &flow_cutoff, &dpi_seq, &tls_seq, &dns_port, &l2_offset, &profile)) return NULL; #endif if (l2_offset < 0 || l2_offset > 64) { PyBuffer_Release(&view); PyErr_SetString(PyExc_ValueError, "l2_offset out of range [0,64]"); return NULL; } if (profile != 0 && profile != 1) { PyBuffer_Release(&view); PyErr_SetString(PyExc_ValueError, "profile must be 0 or 1"); return NULL; } int perr = 0; unsigned int ndpi = 0, ntls = 0; unsigned short* dpi_ports = seq_to_ports(dpi_seq, &ndpi, &perr); if (perr) { PyBuffer_Release(&view); free(dpi_ports); return NULL; } unsigned short* tls_ports = seq_to_ports(tls_seq, &ntls, &perr); if (perr) { PyBuffer_Release(&view); free(dpi_ports); free(tls_ports); return NULL; } int dpi_given = (dpi_seq && dpi_seq != Py_None); int tls_given = (tls_seq && tls_seq != Py_None); struct BufferCtx b; b.buf = (unsigned char*)view.buf; b.cap = (size_t)view.len; b.used = 0; b.admit_mask = admit_mask; b.cfg.dns_port = (unsigned short)dns_port; b.cfg.dpi_ports = dpi_given ? dpi_ports : MT_DPI_PORTS_DEFAULT; b.cfg.ndpi = dpi_given ? ndpi : (unsigned)(sizeof(MT_DPI_PORTS_DEFAULT)/sizeof(MT_DPI_PORTS_DEFAULT[0])); b.cfg.tls_ports = tls_given ? tls_ports : MT_TLS_PORTS_DEFAULT; b.cfg.ntls = tls_given ? ntls : (unsigned)(sizeof(MT_TLS_PORTS_DEFAULT)/sizeof(MT_TLS_PORTS_DEFAULT[0])); b.cfg.l2_off = (unsigned)l2_offset; b.cfg.profile = profile; b.ioc = (iocbuf && ioclen >= 4) ? ipset_build((const unsigned char*)iocbuf, ioclen) : NULL; b.flow_cutoff = (flow_cutoff > 0) ? (unsigned)flow_cutoff : 0; b.flowtab = b.flow_cutoff ? flowtab_build(20) : NULL; b.written = b.dropped = b.overflow = b.cls[0] = b.cls[1] = b.cls[2] = b.cls[3] = b.cls[4] = b.cls[5] = 0; Py_BEGIN_ALLOW_THREADS /* GIL released for the ENTIRE loop — no per-pkt crossing */ pcap_loop(pp->pcap, cant, BufferCallBack, (u_char*)&b); Py_END_ALLOW_THREADS ipset_free(b.ioc); flowtab_free(b.flowtab); free(dpi_ports); free(tls_ports); PyBuffer_Release(&view); /* generic -> 7-tuple (written,dropped,overflow,used, other,flow_head,set_match); security -> 10-tuple (... + the 6 security class counts) */ if (profile == 1) return Py_BuildValue("(KKKKKKKKKK)", b.written, b.dropped, b.overflow, (unsigned long long)b.used, b.cls[0], b.cls[1], b.cls[2], b.cls[3], b.cls[4], b.cls[5]); return Py_BuildValue("(KKKKKKK)", b.written, b.dropped, b.overflow, (unsigned long long)b.used, b.cls[0], b.cls[1], b.cls[2]); } /* ---- F1: batched delivery — N packets per call (one buffer + index), amortize the boundary ---- */ struct BatchMeta { unsigned int sec, usec, off, len; }; struct BatchCtx { unsigned char* buf; size_t cap, used; struct BatchMeta* meta; unsigned int mcap, n; }; static void BatchCallBack(u_char* user, const struct pcap_pkthdr* h, const u_char* data) { struct BatchCtx* b = (struct BatchCtx*)user; unsigned int cl = h->caplen; if (b->used + cl > b->cap) { size_t nc = b->cap * 2; if (nc < b->used + cl + 4096) nc = b->used + cl + 4096; unsigned char* nb = (unsigned char*)realloc(b->buf, nc); if (!nb) return; b->buf = nb; b->cap = nc; } if (b->n >= b->mcap) { unsigned int nm = b->mcap * 2; if (nm == 0) nm = 256; struct BatchMeta* nmeta = (struct BatchMeta*)realloc(b->meta, nm * sizeof(struct BatchMeta)); if (!nmeta) return; b->meta = nmeta; b->mcap = nm; } memcpy(b->buf + b->used, data, cl); b->meta[b->n].sec = (unsigned int)h->ts.tv_sec; b->meta[b->n].usec = (unsigned int)h->ts.tv_usec; b->meta[b->n].off = (unsigned int)b->used; b->meta[b->n].len = cl; b->used += cl; b->n++; } static PyObject* p_next_batch(pcapobject* pp, PyObject* args) { int max_n = 256; if (Py_TYPE(pp) != &Pcaptype) { PyErr_SetString(PcapError, "Not a pcap object"); return NULL; } if (!pp->pcap) return err_closed(); if (!PyArg_ParseTuple(args, "i:next_batch", &max_n)) return NULL; if (max_n <= 0) max_n = 256; struct BatchCtx b; b.cap = 65536; b.used = 0; b.mcap = (unsigned)max_n; b.n = 0; b.buf = (unsigned char*)malloc(b.cap); b.meta = (struct BatchMeta*)malloc(b.mcap * sizeof(struct BatchMeta)); if (!b.buf || !b.meta) { free(b.buf); free(b.meta); return PyErr_NoMemory(); } PyThreadState* ts = PyEval_SaveThread(); int ret = pcap_dispatch(pp->pcap, max_n, BatchCallBack, (u_char*)&b); PyEval_RestoreThread(ts); if (ret < 0 && ret != -2) { free(b.buf); free(b.meta); PyErr_SetString(PcapError, pcap_geterr(pp->pcap)); return NULL; } /* packed meta: N contiguous 16-byte records (sec,usec,off,len as native uint32) -> ONE bytes, no per-packet Python object. Consumer reads via array('I')/struct (records = len(meta)//16). */ PyObject* pybuf = PyBytes_FromStringAndSize((const char*)b.buf, (Py_ssize_t)b.used); PyObject* pymeta = PyBytes_FromStringAndSize((const char*)b.meta, (Py_ssize_t)(b.n * sizeof(struct BatchMeta))); free(b.buf); free(b.meta); if (!pybuf || !pymeta) { Py_XDECREF(pybuf); Py_XDECREF(pymeta); return PyErr_NoMemory(); } return Py_BuildValue("(NN)", pybuf, pymeta); } pcapy-ng-2.1.0/pcapobj.h000077500000000000000000000006611524067405300150330ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #ifndef __pcapobj__ #define __pcapobj__ PyObject* new_pcapobject(pcap_t *pcap, bpf_u_int32 net=0, bpf_u_int32 mask=0); extern PyTypeObject Pcaptype; #endif // __pcapobj__ pcapy-ng-2.1.0/pcapy.cc000077500000000000000000000373771524067405300147050ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #define PY_SSIZE_T_CLEAN #include #include #ifdef WIN32 #include // IPHelper API for luid/guid conversions #endif #if defined(__linux__) #include // PACKET_FANOUT_* mode constants #endif #include "pcapy.h" #include "pcapobj.h" #include "bpfobj.h" #include "pcapdumper.h" #include "pcap_pkthdr.h" PyObject *PcapError; // module methods static PyObject* lookupdev(PyObject* self, PyObject* args) { char errbuff[PCAP_ERRBUF_SIZE]; char* dev; dev = pcap_lookupdev(errbuff); if(!dev) { PyErr_SetString(PcapError, errbuff); return NULL; } return Py_BuildValue("s", dev); } static PyObject* findalldevs(PyObject *self, PyObject *args) { char errbuff[PCAP_ERRBUF_SIZE]; pcap_if_t *devs; int status = pcap_findalldevs(&devs, errbuff); if(status) { PyErr_SetString(PcapError, errbuff); return NULL; } if(devs==NULL) { PyErr_SetString(PcapError, "No valid interfaces to open"); return NULL; } pcap_if_t *cursor = devs; PyObject* list = PyList_New(0); while(cursor) { PyObject *tmp_name; #ifdef WIN32 /* If we are on windows, then display LUID's like "Local Area Connection" * instead of GUID's like "\Device\NPF_{AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA}". */ char *luid = get_windows_interface_friendly_name(cursor->name); tmp_name = Py_BuildValue("s", luid); PyList_Append(list, tmp_name); Py_DECREF(tmp_name); // Fix: Decrement reference count free(luid); #else tmp_name = Py_BuildValue("s", cursor->name); PyList_Append(list, tmp_name); Py_DECREF(tmp_name); // Fix: Decrement reference count #endif cursor = cursor->next; } pcap_freealldevs(devs); return list; } static PyObject* open_live(PyObject *self, PyObject *args) { char errbuff[PCAP_ERRBUF_SIZE]; char * device; int snaplen; int promisc; int to_ms; bpf_u_int32 net, mask; if(!PyArg_ParseTuple(args,"siii:open_live",&device,&snaplen,&promisc,&to_ms)) return NULL; #ifdef WIN32 /* If we are on windows, then device is an LUID like "Local Area Connection". * pcap on Windows needs a GUID like "\Device\NPF_{AAAAAAAA-AAAA-AAAA-AAAA-AAAAAAAAAAAA}". */ char *old_device = device; // Keep track of whether we switched pointers device = luid_to_guid(device); // luid_to_guid returns a new allocated string. // If it fails, it returns NULL, which we must handle. if (device == NULL) { // Fallback or error handling if needed, // though the original code didn't handle NULL return well here. device = old_device; } #endif int status = pcap_lookupnet(device, &net, &mask, errbuff); if(status) { net = 0; mask = 0; } pcap_t* pt; pt = pcap_open_live(device, snaplen, promisc!=0, to_ms, errbuff); #ifdef WIN32 if (device != old_device) free(device); #endif if(!pt) { PyErr_SetString(PcapError, errbuff); return NULL; } #ifdef WIN32 //According to the doc // pcap_setmintocopy() changes the minimum amount of data in the kernel buffer that causes a read from the application to return (unless the timeout expires) // [...] pcap_open_live() sets a default mintocopy value of 16000 bytes. //It is a better practice to set it to 0, so that we are transparent about what we receive pcap_setmintocopy(pt, 0); #endif return new_pcapobject( pt, net, mask ); } static PyObject* pcap_create(PyObject *self, PyObject *args) { char errbuff[PCAP_ERRBUF_SIZE]; char * device; bpf_u_int32 net, mask; if (!PyArg_ParseTuple(args, "s:pcap_create", &device)) return NULL; int status = pcap_lookupnet(device, &net, &mask, errbuff); if (status) { net = 0; mask = 0; } pcap_t* pt; pt = pcap_create(device, errbuff); if (!pt) { PyErr_SetString(PcapError, errbuff); return NULL; } #ifdef WIN32 //Same than in open_live pcap_setmintocopy(pt, 0); #endif return new_pcapobject(pt, net, mask); } static PyObject* open_offline(PyObject *self, PyObject *args) { char errbuff[PCAP_ERRBUF_SIZE]; char * filename; if(!PyArg_ParseTuple(args,"s",&filename)) return NULL; pcap_t* pt; pt = pcap_open_offline(filename, errbuff); if(!pt) { PyErr_SetString(PcapError, errbuff); return NULL; } #ifdef WIN32 pcap_setmintocopy(pt, 0); #endif return new_pcapobject( pt ); } static PyObject* bpf_compile(PyObject* self, PyObject* args) { int linktype; int snaplen; char *filter; int optimize; unsigned int netmask; if(!PyArg_ParseTuple(args, "iisiI:compile", &linktype, &snaplen, &filter, &optimize, &netmask)) return NULL; pcap_t *pp; pp = pcap_open_dead(linktype, snaplen); if(pp == NULL) return NULL; struct bpf_program bpf; int status = pcap_compile(pp, &bpf, filter, optimize, netmask); pcap_close(pp); if(status) { PyErr_SetString(PcapError, pcap_geterr(pp)); return NULL; } return new_bpfobject( bpf ); } static PyMethodDef pcap_methods[] = { {"open_live", open_live, METH_VARARGS, "open_live(device, snaplen, promisc, to_ms) opens a pcap device"}, {"open_offline", open_offline, METH_VARARGS, "open_offline(filename) opens a pcap formated file"}, {"lookupdev", lookupdev, METH_VARARGS, "lookupdev() looks up a pcap device"}, {"findalldevs", findalldevs, METH_VARARGS, "findalldevs() lists all available interfaces"}, {"compile", bpf_compile, METH_VARARGS, "compile(linktype, snaplen, filter, optimize, netmask) creates a bpfprogram object"}, {"create", pcap_create, METH_VARARGS, "create(device) is used to create a packet capture handle to look at packets on the network."}, {NULL, NULL} }; #if PY_MAJOR_VERSION >= 3 PyDoc_STRVAR(pcap_doc, "A wrapper for the Packet Capture (PCAP) library"); static struct PyModuleDef pcapy_module = { PyModuleDef_HEAD_INIT, "pcapy", /* m_name */ pcap_doc, /* m_doc */ -1, /* m_size */ pcap_methods, /* m_methods */ NULL, /* m_reload */ NULL, /* m_traverse */ NULL, /* m_clear */ NULL, /* m_free */ }; #else static char *pcap_doc = "\nA wrapper for the Packet Capture (PCAP) library\n"; #endif //PY_MAJOR_VERSION >= 3 #if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit_pcapy(void) #else void initpcapy(void) #endif //PY_MAJOR_VERSION >= 3 { PyObject *m, *d; #if PY_MAJOR_VERSION < 3 Pcaptype.ob_type = &PyType_Type; Pkthdr_type.ob_type = &PyType_Type; Pdumpertype.ob_type = &PyType_Type; #endif #if PY_MAJOR_VERSION >= 3 m = PyModule_Create(&pcapy_module); #else m = Py_InitModule3("pcapy", pcap_methods, pcap_doc); #endif if (PyType_Ready(&BPFProgramType) < 0) { #if PY_MAJOR_VERSION >= 3 return NULL; #else return; #endif //PY_MAJOR_VERSION >= 3 } PyModule_AddObject(m, "BPFProgram", (PyObject *) &BPFProgramType); /* Direct from pcap's net/bpf.h. */ PyModule_AddIntConstant(m, "DLT_NULL", 0); PyModule_AddIntConstant(m, "DLT_EN10MB", 1); PyModule_AddIntConstant(m, "DLT_IEEE802", 6); PyModule_AddIntConstant(m, "DLT_ARCNET", 7); PyModule_AddIntConstant(m, "DLT_SLIP", 8); PyModule_AddIntConstant(m, "DLT_PPP", 9); PyModule_AddIntConstant(m, "DLT_FDDI", 10); PyModule_AddIntConstant(m, "DLT_ATM_RFC1483", 11); PyModule_AddIntConstant(m, "DLT_RAW", 12); PyModule_AddIntConstant(m, "DLT_PPP_SERIAL", 50); PyModule_AddIntConstant(m, "DLT_PPP_ETHER", 51); PyModule_AddIntConstant(m, "DLT_C_HDLC", 104); PyModule_AddIntConstant(m, "DLT_IEEE802_11", 105); PyModule_AddIntConstant(m, "DLT_LOOP", 108); PyModule_AddIntConstant(m, "DLT_LINUX_SLL", 113); PyModule_AddIntConstant(m, "DLT_LTALK", 114); /* Direct from pcap's net/pcap.h.*/ PyModule_AddIntConstant(m, "PCAP_D_INOUT", 0); PyModule_AddIntConstant(m, "PCAP_D_IN", 1); PyModule_AddIntConstant(m, "PCAP_D_OUT", 2); /* PACKET_FANOUT modes for Reader.set_fanout() (Linux AF_PACKET; from linux/if_packet.h). */ #if defined(__linux__) && defined(PACKET_FANOUT) PyModule_AddIntConstant(m, "PACKET_FANOUT_HASH", PACKET_FANOUT_HASH); PyModule_AddIntConstant(m, "PACKET_FANOUT_LB", PACKET_FANOUT_LB); PyModule_AddIntConstant(m, "PACKET_FANOUT_CPU", PACKET_FANOUT_CPU); #ifdef PACKET_FANOUT_ROLLOVER PyModule_AddIntConstant(m, "PACKET_FANOUT_ROLLOVER", PACKET_FANOUT_ROLLOVER); #endif #ifdef PACKET_FANOUT_RND PyModule_AddIntConstant(m, "PACKET_FANOUT_RND", PACKET_FANOUT_RND); #endif #endif d = PyModule_GetDict(m); PcapError = PyErr_NewException("pcapy.PcapError", NULL, NULL ); BPFError = PyErr_NewException("pcapy.BPFError", NULL, NULL ); if( PcapError ) { PyDict_SetItemString( d, "PcapError", PcapError ); } if ( BPFError ) { PyDict_SetItemString( d, "BPFError", BPFError ); } #if PY_MAJOR_VERSION >= 3 return m; #endif //PY_MAJOR_VERSION >= 3 } /* vim: set tabstop=2 shiftwidth=2 expandtab: */ #ifdef WIN32 /* All of the following code takes care of LUID/GUID conversions. * Copied from Wireshark. */ static BOOL gethexdigit(const char *p) { if(*p >= '0' && *p <= '9'){ return *p - '0'; }else if(*p >= 'A' && *p <= 'F'){ return *p - 'A' + 0xA; }else if(*p >= 'a' && *p <= 'f'){ return *p - 'a' + 0xa; }else{ return -1; /* Not a hex digit */ } } static BOOL get8hexdigits(const char *p, DWORD *d) { int digit; DWORD val; int i; val = 0; for(i = 0; i < 8; i++){ digit = gethexdigit(p++); if(digit == -1){ return FALSE; /* Not a hex digit */ } val = (val << 4) | digit; } *d = val; return TRUE; } static BOOL get4hexdigits(const char *p, WORD *w) { int digit; WORD val; int i; val = 0; for(i = 0; i < 4; i++){ digit = gethexdigit(p++); if(digit == -1){ return FALSE; /* Not a hex digit */ } val = (val << 4) | digit; } *w = val; return TRUE; } static BOOL parse_as_guid(const char *guid_text, GUID *guid) { int i; int digit1, digit2; if(*guid_text != '{'){ return FALSE; /* Nope, not enclosed in {} */ } guid_text++; /* There must be 8 hex digits; if so, they go into guid->Data1 */ if(!get8hexdigits(guid_text, &guid->Data1)){ return FALSE; /* nope, not 8 hex digits */ } guid_text += 8; /* Now there must be a hyphen */ if(*guid_text != '-'){ return FALSE; /* Nope */ } guid_text++; /* There must be 4 hex digits; if so, they go into guid->Data2 */ if(!get4hexdigits(guid_text, &guid->Data2)){ return FALSE; /* nope, not 4 hex digits */ } guid_text += 4; /* Now there must be a hyphen */ if(*guid_text != '-'){ return FALSE; /* Nope */ } guid_text++; /* There must be 4 hex digits; if so, they go into guid->Data3 */ if(!get4hexdigits(guid_text, &guid->Data3)){ return FALSE; /* nope, not 4 hex digits */ } guid_text += 4; /* Now there must be a hyphen */ if(*guid_text != '-'){ return FALSE; /* Nope */ } guid_text++; /* * There must be 4 hex digits; if so, they go into the first 2 bytes * of guid->Data4. */ for(i = 0; i < 2; i++){ digit1 = gethexdigit(guid_text); if(digit1 == -1){ return FALSE; /* Not a hex digit */ } guid_text++; digit2 = gethexdigit(guid_text); if(digit2 == -1){ return FALSE; /* Not a hex digit */ } guid_text++; guid->Data4[i] = (digit1 << 4)|(digit2); } /* Now there must be a hyphen */ if(*guid_text != '-'){ return FALSE; /* Nope */ } guid_text++; /* * There must be 12 hex digits; if so,t hey go into the next 6 bytes * of guid->Data4. */ for(i = 0; i < 6; i++){ digit1 = gethexdigit(guid_text); if(digit1 == -1){ return FALSE; /* Not a hex digit */ } guid_text++; digit2 = gethexdigit(guid_text); if(digit2 == -1){ return FALSE; /* Not a hex digit */ } guid_text++; guid->Data4[i+2] = (digit1 << 4)|(digit2); } /* Now there must be a closing } */ if(*guid_text != '}'){ return FALSE; /* Nope */ } guid_text++; /* And that must be the end of the string */ if(*guid_text != '\0'){ return FALSE; /* Nope */ } return TRUE; } static char* luid_to_guid(char *luid) { char errbuff[PCAP_ERRBUF_SIZE]; pcap_if_t *devs; int status = pcap_findalldevs(&devs, errbuff); if(status) { PyErr_SetString(PcapError, errbuff); return NULL; } if(devs==NULL) { PyErr_SetString(PcapError, "No valid interfaces to open"); return NULL; } pcap_if_t *cursor = devs; char* cursor_luid; char *ret_guid = NULL; while(cursor) { cursor_luid = get_windows_interface_friendly_name(cursor->name); if (!strcmp(cursor_luid, luid)) { ret_guid = strdup(cursor->name); goto done; } free(cursor_luid); cursor = cursor->next; } done: pcap_freealldevs(devs); return ret_guid; } static char* guid_to_luid(GUID *guid) { BOOL status; int size; char *name; WCHAR wName[128 + 1]; HMODULE hIPHlpApi = LoadLibrary(TEXT("iphlpapi.dll")); typedef HRESULT (WINAPI *ProcAddr_nhGINFG) ( GUID *InterfaceGuid, PCWSTR InterfaceAlias, DWORD *LengthAddress, wchar_t *a4, wchar_t *a5); ProcAddr_nhGINFG Proc_nhGetInterfaceNameFromGuid = NULL; Proc_nhGetInterfaceNameFromGuid = (ProcAddr_nhGINFG) GetProcAddress(hIPHlpApi, "NhGetInterfaceNameFromGuid"); if (Proc_nhGetInterfaceNameFromGuid!= NULL) { wchar_t *p4=NULL, *p5=NULL; DWORD NameSize; /* testing of nhGetInterfaceNameFromGuid indicates the unpublished API function expects the 3rd parameter * to be the available space in bytes (as compared to wchar's) available in the second parameter buffer * to receive the friendly name (in unicode format) including the space for the nul termination.*/ NameSize = sizeof(wName); /* do the guid->friendlyname lookup */ status = ( 0 == Proc_nhGetInterfaceNameFromGuid(guid, wName, &NameSize, p4, p5) ); } /* we have finished with iphlpapi.dll - release it */ FreeLibrary(hIPHlpApi); if(FALSE == status){ /* failed to get the friendly name, nothing further to do */ return NULL; } /* Get the required buffer size, and then convert the string * from UTF-16 to UTF-8. */ size=WideCharToMultiByte(CP_UTF8, 0, wName, -1, NULL, 0, NULL, NULL); name=(char *) malloc(size); if (name == NULL){ return NULL; } size=WideCharToMultiByte(CP_UTF8, 0, wName, -1, name, size, NULL, NULL); if(size==0){ /* bytes written == 0, indicating some form of error*/ free(name); return NULL; } return name; } char * get_windows_interface_friendly_name(const char *interface_devicename) { const char* guid_text; GUID guid; /* Extract the guid text from the interface device name */ if(strncmp("\\Device\\NPF_", interface_devicename, 12)==0){ guid_text=interface_devicename+12; /* skip over the '\Device\NPF_' prefix, assume the rest is the guid text */ }else{ guid_text=interface_devicename; } if (!parse_as_guid(guid_text, &guid)){ return strdup(interface_devicename); /* not a GUID, so no friendly name */ } /* guid okay, get the interface friendly name associated with the guid */ return guid_to_luid(&guid); } #endif pcapy-ng-2.1.0/pcapy.h000077500000000000000000000013211524067405300145230ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * */ #ifndef __PCAPY_H__ #ifndef Py_TYPE // python3 compatible #define Py_TYPE(ob) (((PyObject*)(ob))->ob_type) #endif extern "C" { #ifdef WIN32 __declspec(dllexport) char *get_windows_interface_friendly_name(const char *interface_devicename); static char* luid_to_guid(char *luid); #endif #if PY_MAJOR_VERSION >= 3 PyMODINIT_FUNC PyInit_pcapy(void); #else void initpcapy(void); #endif } // exception object extern PyObject* PcapError; #endif // __PCAPY_H__ pcapy-ng-2.1.0/pcapy.html000077500000000000000000001751341524067405300152560ustar00rootroot00000000000000 Part I. Pcapy Reference

Part I. Pcapy Reference

CORE SECURITY TECHNOLOGIES

Copyright © 2016-2018 CORE SECURITY TECHNOLOGIES

Revision History
Revision Revision: 16Date: 2018-06-04Author: jkohen
2018 updated

Table of Contents

I. Pcapy Module Reference
open_live — Obtain a packet capture descriptor to look at packets on the network
open_offline — Obtain a packet capture descriptor to look at packets on a savefile
lookupdev — Return a network device suitable for use with open_live
findalldevs — Obtain the list of available network devices
compile — Compile a BPF filter
create — Creates a non-activated packet capture handle to look at packets on the network
II. Reader Object Reference
dispatch — Collect and process packets
next — Collect the next packet
stats — get capture statistics
setfilter — Specify a filter
getfd — get a file descriptor on which a select() can be done for a live capture
set_snaplen — Set the snapshot length for a not-yet-activated capture handle
set_promisc — Set promiscuous mode for a not-yet-activated capture handle
set_timeout — Set the read timeout for a not-yet-activated capture handle
set_buffer_size — Set the buffer size for a not-yet-activated capture handle capture handle
activate — Activate a capture handle
getnet — Get the associated network number and mask
datalink — Obtain the link layer type
getnonblock / setnonblock — Manipulate the non-blocking flag
dump_open — Create a Dumper object
close — Close a Reader
III. Dumper Object Reference
dump — Dump a packet to a savefile
close — Close a Dumper
IV. Pkthdr Object Reference
getts — Obtain packet header information
V. Bpf Object Reference
filter — Test a packet against a compiled filter
Bibliography

Pcapy Module Reference

Table of Contents

open_live — Obtain a packet capture descriptor to look at packets on the network
open_offline — Obtain a packet capture descriptor to look at packets on a savefile
lookupdev — Return a network device suitable for use with open_live
findalldevs — Obtain the list of available network devices
compile — Compile a BPF filter
create — Creates a non-activated packet capture handle to look at packets on the network

Name

open_live — Obtain a packet capture descriptor to look at packets on the network

Synopsis

Reader open_live (device,  
 snaplen,  
 promisc,  
 to_ms); 
string device ;
int snaplen ;
int promisc ;
int to_ms ;
 

DESCRIPTION

open_live is used to obtain a packet capture descriptor to look at packets on the network. device is a string that specifies the network device to open; on Linux systems with 2.2 or later kernels, a device argument of any or NULL can be used to capture packets from all interfaces. snaplen specifies the maximum number of bytes to capture. promisc specifies if the interface is to be put into promiscuous mode. (Note that even if this parameter is false, the interface could well be in promiscuous mode for some other reason.) For now, this doesn't work on the any device; if an argument of any or NULL is supplied, the promisc flag is ignored. to_ms specifies the read timeout in milliseconds. The read timeout is used to arrange that the read not necessarily return immediately when a packet is seen, but that it wait for some amount of time to allow more packets to arrive and to read multiple packets from the OS kernel in one operation. Not all platforms support a read timeout; on platforms that don't, the read timeout is ignored.

Name

open_offline — Obtain a packet capture descriptor to look at packets on a savefile

Synopsis

Reader open_offline (filename); 
string filename ;
 

DESCRIPTION

open_offline is called to open a savefile for reading. filename specifies the name of the file to open. The file has the same format as those used by tcpdump(8) and tcpslice(8). The name - is a synonym for stdin.

Name

lookupdev — Return a network device suitable for use with open_live

Synopsis

string lookupdev (); 
 

DESCRIPTION

lookupdev returns the name of a network device suitable for use with open_live.

Name

findalldevs — Obtain the list of available network devices

Synopsis

string[] findalldevs (); 
 

DESCRIPTION

findalldevs constructs a list of network devices that can be opened with open_live. (Note that there may be network devices that cannot be opened with open_live, because, for example, that process might not have sufficient privileges to open them for capturing; if so, those devices will not appear on the list.)

Name

compile — Compile a BPF filter

Synopsis

Bpf compile (linktype,  
 snaplen,  
 filter,  
 optimize,  
 netmask); 
int linktype ;
int snaplen ;
string filter ;
int optimize ;
int32 netmask ;
 

DESCRIPTION

compile is used to compile the filter into a filter program. snaplen specifies the maximum number of bytes to capture. optimize controls whether optimization on the resulting code is performed. netmask specifies the netmask of the local network.

Name

create — Creates a non-activated packet capture handle to look at packets on the network

Synopsis

Reader create (device); 
string device ;
 

DESCRIPTION

create is used to create a packet capture handle to look at packets on the network. The returned handle must be activated with activate() before packets can be captured with it; options for the capture, such as promiscuous mode, can be set on the handle before activating it.

Reader Object Reference

Table of Contents

dispatch — Collect and process packets
next — Collect the next packet
stats — get capture statistics
setfilter — Specify a filter
getfd — get a file descriptor on which a select() can be done for a live capture
set_snaplen — Set the snapshot length for a not-yet-activated capture handle
set_promisc — Set promiscuous mode for a not-yet-activated capture handle
set_timeout — Set the read timeout for a not-yet-activated capture handle
set_buffer_size — Set the buffer size for a not-yet-activated capture handle capture handle
activate — Activate a capture handle
getnet — Get the associated network number and mask
datalink — Obtain the link layer type
getnonblock / setnonblock — Manipulate the non-blocking flag
dump_open — Create a Dumper object
close — Close a Reader

Name

dispatch, loop — Collect and process packets

Synopsis

int dispatch (maxcant,  
 (* callback)); 
int maxcant ;
void (* callback) (Pkthdr, string) ;
 
int loop (maxcant,  
 (* callback)); 
int maxcant ;
void (* callback) (Pkthdr, string) ;
 

DESCRIPTION

dispatch is used to collect and process packets. maxcant specifies the maximum number of packets to process before returning. This is not a minimum number; when reading a live capture, only one bufferful of packets is read at a time, so fewer than maxcant packets may be processed. A cnt of -1 processes all the packets received in one buffer when reading a live capture, or all the packets in the file when reading a savefile. callback specifies a routine to be called with two arguments: a Pkthdr instance describing the data passed and the data itself.

The number of packets read is returned. 0 is returned if no packets were read from a live capture (if, for example, they were discarded because they didn't pass the packet filter, or if, on platforms that support a read timeout that starts before any packets arrive, the timeout expires before any packets arrive, or if the file descriptor for the capture device is in non-blocking mode and no packets were available to be read) or if no more packets are available in a savefile.

Note

When reading a live capture, dispatch will not necessarily return when the read times out; on some platforms, the read timeout isn't supported, and, on other platforms, the timer doesn't start until at least one packet arrives. This means that the read timeout should not be used in, for example, an interactive application, to allow the packet capture loop to poll for user input periodically, as there's no guarantee that dispatch will return after the timeout expires.

loop is similar to dispatch except it keeps reading packets until maxcant packets are processed or an error occurs. It does not return when live read timeouts occur. Rather, specifying a non-zero read timeout to open_live and then calling dispatch allows the reception and processing of any packets that arrive when the timeout occurs. A negative maxcant causes loop to loop forever (or at least until an error occurs). 0 is returned if maxcant is exhausted.

Name

next — Collect the next packet

Synopsis

(Pkthdr, string) next (); 
 

DESCRIPTION

next reads the next packet (by calling dispatch with a maxcant of 1) and returns a tuple (header, data) where header is a Pkthdr instance describing the data passed and data is the data itself.

Name

stats — get capture statistics

Synopsis

(int32, int32, int32) stats (); 
 

DESCRIPTION

stats returns statistics on the current capture as a tuple (recv, drop, ifdrop)

Name

setfilter — Specify a filter

Synopsis

setfilter (filter); 
string filter ;
 

DESCRIPTION

setfilter is used to specify a filter for this object.

Name

getfd — get a file descriptor on which a select() can be done for a live capture

Synopsis

int getfd (filter); 
string filter ;
 

DESCRIPTION

getfd returns, on UNIX, a file descriptor number for a file descriptor on which one can do a select(), poll(), epoll_wait(), kevent(), or other such call to wait for it to be possible to read packets without blocking, if such a descriptor exists, or -1, if no such descriptor exists.

Name

set_snaplen — Set the snapshot length for a not-yet-activated capture handle

Synopsis

int set_snaplen (snaplen); 
int snaplen ;
 

DESCRIPTION

set_snaplen sets the snapshot length to be used on a capture handle when the handle is activated to snaplen. set_snaplen returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated.

Name

set_promisc — Set promiscuous mode for a not-yet-activated capture handle

Synopsis

int set_promisc (promisc); 
int promisc ;
 

DESCRIPTION

set_promisc sets whether promiscuous mode should be set on a capture handle when the handle is activated. If promisc is non-zero, promiscuous mode will be set, otherwise it will not be set. set_promisc returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated.

Name

set_timeout — Set the read timeout for a not-yet-activated capture handle

Synopsis

int set_timeout (timeout); 
int timeout ;
 

DESCRIPTION

set_timeout sets the read timeout that will be used on a capture handle when the handle is activated to to_ms, which is in units of milliseconds. set_timeout returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated.

Name

set_buffer_size — Set the buffer size for a not-yet-activated capture handle capture handle

Synopsis

int set_buffer_size (buffer_size); 
int buffer_size ;
 

DESCRIPTION

set_buffer_size sets the buffer size that will be used on a capture handle when the handle is activated to buffer_size, which is in units of bytes. set_buffer_size returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated.

Name

activate — Activate a capture handle

Synopsis

int activate (); 
 

DESCRIPTION

activate is used to activate a packet capture handle to look at packets on the network, with the options that were set on the handle being in effect. activate returns 0 on success without warnings, a non-zero positive value on success with warnings, and a negative value on error. A non-zero return value indicates what warning or error condition occurred. has been activated. See https://www.tcpdump.org/manpages/pcap_activate.3pcap.html for all possible return values.

Name

getnet, getmask — Get the associated network number and mask

Synopsis

int32 getnet (); 
 
int32 getmask (); 
 

DESCRIPTION

getnet and getmask are used to determine the network number and mask associated with the network device attached to this Reader.

Name

datalink — Obtain the link layer type

Synopsis

int datalink (); 
 

DESCRIPTION

datalink returns the link layer type; link layer types it can return include:

DLT_NULL

BSD loopback encapsulation; the link layer header is a 4-byte field, in host byte order, containing a PF_ value from socket.h for the network-layer protocol of the packet.

Note

“host byte order” is the byte order of the machine on which the packets are captured, and the PF_ values are for the OS of the machine on which the packets are captured; if a live capture is being done, “host byte order” is the byte order of the machine capturing the packets, and the PF_ values are those of the OS of the machine capturing the packets, but if a savefile is being read, the byte order and PF_ values are not necessarily those of the machine reading the capture file.

DLT_EN10MB
Ethernet (10Mb, 100Mb, 1000Mb, and up)
DLT_IEEE802
IEEE 802.5 Token Ring
DLT_ARCNET
ARCNET
DLT_SLIP

SLIP; the link layer header contains, in order:

  • a 1-byte flag, which is 0 for packets received by the machine and 1 for packets sent by the machine.

  • a 1-byte field, the upper 4 bits of which indicate the type of packet, as per RFC 1144:

    • 0x40; an unmodified IP datagram (TYPE_IP)
    • 0x70; an uncompressed-TCP/IP datagram (UNCOMPRESSED_TCP), with that byte being the first byte of the raw IP header on the wire, containing the connection number in the protocol field
    • 0x80; a compressed-TCP/IP datagram (COMPRESSED_TCP), with that byte being the first byte of the compressed TCP/IP datagram header

  • for UNCOMPRESSED_TCP, the rest of the modified IP header, and for COMPRESSED_TCP, the compressed TCP/IP datagram header

for a total of 16 bytes; the uncompressed IP datagram follows the header.

DLT_PPP
PPP; if the first 2 bytes are 0xff and 0x03, it's PPP in HDLC-like framing, with the PPP header following those two bytes, otherwise it's PPP without framing, and the packet begins with the PPP header.
DLT_FDDI
FDDI
DLT_ATM_RFC1483
RFC 1483 LLC/SNAP-encapsulated ATM; the packet begins with an IEEE 802.2 LLC header.
DLT_RAW
Raw IP; the packet begins with an IP header.
DLT_PPP_SERIAL
PPP in HDLC-like framing, as per RFC 1662, or Cisco PPP with HDLC framing, as per section 4.3.1 of RFC 1547; the first byte will be 0xFF for PPP in HDLC-like framing, and will be 0x0F or 0x8F for Cisco PPP with HDLC framing.
DLT_PPP_ETHER
PPPoE; the packet begins with a PPPoE header, as per RFC 2516.
DLT_C_HDLC
Cisco PPP with HDLC framing, as per section 4.3.1 of RFC 1547.
DLT_IEEE802_11
IEEE 802.11 wireless LAN.
DLT_LOOP

OpenBSD loopback encapsulation; the link layer header is a 4-byte field, in network byte order, containing a PF_ value from OpenBSD's socket.h for the network-layer protocol of the packet.

Note

Note that, if a savefile is being read, those PF_ values are not necessarily those of the machine reading the capture file.

DLT_LINUX_SLL

Linux cooked capture encapsulation; the link layer header contains, in order:

  • a 2-byte "packet type", in network byte order, which is one of:

    • 0; packet was sent to us by somebody else.
    • 1; packet was broadcast by somebody else.
    • 2; packet was multicast, but not broadcast, by somebody else.
    • 3; packet was sent by somebody else to somebody else.
    • 4; packet was sent by us.

  • a 2-byte field, in network byte order, containing a Linux ARPHRD_ value for the link layer device type.
  • a 2-byte field, in network byte order, containing the length of the link layer address of the sender of the packet (which could be 0).
  • an 8-byte field containing that number of bytes of the link layer header (if there are more than 8 bytes, only the first 8 are present).
  • a 2-byte field containing an Ethernet protocol type, in network byte order, or containing 1 for Novell 802.3 frames without an 802.2 LLC header or 4 for frames beginning with an 802.2 LLC header.

DLT_LTALK
Apple LocalTalk; the packet begins with an AppleTalk LLAP header.

Name

getnonblock / setnonblock — Manipulate the non-blocking flag

Synopsis

int getnonblock (); 
 
setnonblock (state); 
int state ;
 

DESCRIPTION

getnonblock returns the current non-blocking state of the capture descriptor; it always returns 0 on savefiles.

DESCRIPTION

setnonblock puts a capture descriptor, opened with open_live, into non-blocking mode, or takes it out of non-blocking mode, depending on whether the state argument is non-zero or zero. It has no effect on savefiles. In non-blocking mode, an attempt to read from the capture descriptor with dispatch will, if no packets are currently available to be read, return 0 immediately rather than blocking waiting for packets to arrive. loop and next will not work in non-blocking mode.

Name

dump_open — Create a Dumper object

Synopsis

Dumper dump_open (filename); 
string filename ;
 

DESCRIPTION

dump_open is called to open a savefile for writing and associate it to a newly created Dumper instance. The name - is a synonym for stdout. filename specifies the name of the file to open.

Name

close — Close a Reader

Synopsis

Reader close (); 
 

DESCRIPTION

close closes a Reader using pcap_close.

Dumper Object Reference

Table of Contents

dump — Dump a packet to a savefile
close — Close a Dumper

Name

dump — Dump a packet to a savefile

Synopsis

dump (header,  
 data); 
Pkthdr header ;
string data ;
 

DESCRIPTION

dump outputs a packet to the savefile opened with dump_open from type Reader.

Name

close — Close a Dumper

Synopsis

Dumper close (); 
 

DESCRIPTION

close closes a Dumper.

Pkthdr Object Reference

Table of Contents

getts — Obtain packet header information

Name

getts, getcaplen, getlen — Obtain packet header information

Synopsis

(long, long) getts (); 
 
long getcaplen (); 
 
long getlen (); 
 

DESCRIPTION

getts, getcaplen and getlen return the timestamp, capture length and total length fields of the packet header, respectively.

Timestamp is a tuple with two elements: the number of seconds since the Epoch, and the amount of microseconds past the current second. The capture length is the number of bytes of the packet that are available from the capture. Finally, total length gives the length of the packet, in bytes (which might be more than the number of bytes available from the capture, if the length of the packet is larger than the maximum number of bytes to capture).

Bpf Object Reference

Table of Contents

filter — Test a packet against a compiled filter

Name

filter — Test a packet against a compiled filter

Synopsis

int filter (packet); 
string packet ;
 

DESCRIPTION

filter tests a packet against a compiled filter as returned by pcapy's compile. If the packet is allowed to pass through -1 is returned, otherwise filter returns 0.

Bibliography

Sources

Portions of this work based on pcap(3) by the Lawrence Berkeley National Laboratory, University of California, Berkeley, CA. .

pcapy-ng-2.1.0/pcapy.xml000077500000000000000000001344151524067405300151070ustar00rootroot00000000000000 ]> 2016-2018 &corest; &corest; pcap packet capture python Revision: 16 Date: 2018-06-04 Author: jkohen 2018 updated Pcapy Reference Pcapy Module Reference open_live Obtain a packet capture descriptor to look at packets on the network Reader open_live string device int snaplen int promisc int to_ms DESCRIPTION open_live is used to obtain a packet capture descriptor to look at packets on the network. device is a string that specifies the network device to open; on Linux systems with 2.2 or later kernels, a device argument of any or NULL can be used to capture packets from all interfaces. snaplen specifies the maximum number of bytes to capture. promisc specifies if the interface is to be put into promiscuous mode. (Note that even if this parameter is false, the interface could well be in promiscuous mode for some other reason.) For now, this doesn't work on the any device; if an argument of any or NULL is supplied, the promisc flag is ignored. to_ms specifies the read timeout in milliseconds. The read timeout is used to arrange that the read not necessarily return immediately when a packet is seen, but that it wait for some amount of time to allow more packets to arrive and to read multiple packets from the OS kernel in one operation. Not all platforms support a read timeout; on platforms that don't, the read timeout is ignored. open_offline Obtain a packet capture descriptor to look at packets on a savefile Reader open_offline string filename DESCRIPTION open_offline is called to open a savefile for reading. filename specifies the name of the file to open. The file has the same format as those used by tcpdump 8 and tcpslice 8 . The name - is a synonym for stdin. lookupdev Return a network device suitable for use with open_live string lookupdev DESCRIPTION lookupdev returns the name of a network device suitable for use with open_live. findalldevs Obtain the list of available network devices string[] findalldevs DESCRIPTION findalldevs constructs a list of network devices that can be opened with open_live. (Note that there may be network devices that cannot be opened with open_live, because, for example, that process might not have sufficient privileges to open them for capturing; if so, those devices will not appear on the list.) compile Compile a BPF filter Bpf compile int linktype int snaplen string filter int optimize int32 netmask DESCRIPTION compile is used to compile the filter into a filter program. snaplen specifies the maximum number of bytes to capture. optimize controls whether optimization on the resulting code is performed. netmask specifies the netmask of the local network. create Creates a non-activated packet capture handle to look at packets on the network Reader create string device DESCRIPTION create is used to create a packet capture handle to look at packets on the network. The returned handle must be activated with activate() before packets can be captured with it; options for the capture, such as promiscuous mode, can be set on the handle before activating it. Reader Object Reference dispatch loop Collect and process packets int dispatch int maxcant void (* callback) Pkthdr, string int loop int maxcant void (* callback) Pkthdr, string DESCRIPTION dispatch is used to collect and process packets. maxcant specifies the maximum number of packets to process before returning. This is not a minimum number; when reading a live capture, only one bufferful of packets is read at a time, so fewer than maxcant packets may be processed. A cnt of -1 processes all the packets received in one buffer when reading a live capture, or all the packets in the file when reading a savefile. callback specifies a routine to be called with two arguments: a Pkthdr instance describing the data passed and the data itself. The number of packets read is returned. 0 is returned if no packets were read from a live capture (if, for example, they were discarded because they didn't pass the packet filter, or if, on platforms that support a read timeout that starts before any packets arrive, the timeout expires before any packets arrive, or if the file descriptor for the capture device is in non-blocking mode and no packets were available to be read) or if no more packets are available in a savefile. When reading a live capture, dispatch will not necessarily return when the read times out; on some platforms, the read timeout isn't supported, and, on other platforms, the timer doesn't start until at least one packet arrives. This means that the read timeout should not be used in, for example, an interactive application, to allow the packet capture loop to poll for user input periodically, as there's no guarantee that dispatch will return after the timeout expires. loop is similar to dispatch except it keeps reading packets until maxcant packets are processed or an error occurs. It does not return when live read timeouts occur. Rather, specifying a non-zero read timeout to open_live and then calling dispatch allows the reception and processing of any packets that arrive when the timeout occurs. A negative maxcant causes loop to loop forever (or at least until an error occurs). 0 is returned if maxcant is exhausted. next Collect the next packet (Pkthdr, string) next DESCRIPTION next reads the next packet (by calling dispatch with a maxcant of 1) and returns a tuple (header, data) where header is a Pkthdr instance describing the data passed and data is the data itself. stats get capture statistics (int32, int32, int32) stats DESCRIPTION stats returns statistics on the current capture as a tuple (recv, drop, ifdrop) setfilter Specify a filter setfilter string filter DESCRIPTION setfilter is used to specify a filter for this object. getfd get a file descriptor on which a select() can be done for a live capture int getfd string filter DESCRIPTION getfd returns, on UNIX, a file descriptor number for a file descriptor on which one can do a select(), poll(), epoll_wait(), kevent(), or other such call to wait for it to be possible to read packets without blocking, if such a descriptor exists, or -1, if no such descriptor exists. set_snaplen Set the snapshot length for a not-yet-activated capture handle int set_snaplen int snaplen DESCRIPTION set_snaplen sets the snapshot length to be used on a capture handle when the handle is activated to snaplen. set_snaplen returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated. set_promisc Set promiscuous mode for a not-yet-activated capture handle int set_promisc int promisc DESCRIPTION set_promisc sets whether promiscuous mode should be set on a capture handle when the handle is activated. If promisc is non-zero, promiscuous mode will be set, otherwise it will not be set. set_promisc returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated. set_timeout Set the read timeout for a not-yet-activated capture handle int set_timeout int timeout DESCRIPTION set_timeout sets the read timeout that will be used on a capture handle when the handle is activated to to_ms, which is in units of milliseconds. set_timeout returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated. set_buffer_size Set the buffer size for a not-yet-activated capture handle capture handle int set_buffer_size int buffer_size DESCRIPTION set_buffer_size sets the buffer size that will be used on a capture handle when the handle is activated to buffer_size, which is in units of bytes. set_buffer_size returns 0 on success or PCAP_ERROR_ACTIVATED if called on a capture handle that has been activated. activate Activate a capture handle int activate DESCRIPTION activate is used to activate a packet capture handle to look at packets on the network, with the options that were set on the handle being in effect. activate returns 0 on success without warnings, a non-zero positive value on success with warnings, and a negative value on error. A non-zero return value indicates what warning or error condition occurred. has been activated. See https://www.tcpdump.org/manpages/pcap_activate.3pcap.html for all possible return values. getnet getmask Get the associated network number and mask int32 getnet int32 getmask DESCRIPTION getnet and getmask are used to determine the network number and mask associated with the network device attached to this Reader. datalink Obtain the link layer type int datalink DESCRIPTION datalink returns the link layer type; link layer types it can return include: DLT_NULL BSD loopback encapsulation; the link layer header is a 4-byte field, in host byte order, containing a PF_ value from socket.h for the network-layer protocol of the packet. host byte order is the byte order of the machine on which the packets are captured, and the PF_ values are for the OS of the machine on which the packets are captured; if a live capture is being done, host byte order is the byte order of the machine capturing the packets, and the PF_ values are those of the OS of the machine capturing the packets, but if a savefile is being read, the byte order and PF_ values are not necessarily those of the machine reading the capture file. DLT_EN10MB Ethernet (10Mb, 100Mb, 1000Mb, and up) DLT_IEEE802 IEEE 802.5 Token Ring DLT_ARCNET ARCNET DLT_SLIP SLIP; the link layer header contains, in order: a 1-byte flag, which is 0 for packets received by the machine and 1 for packets sent by the machine. a 1-byte field, the upper 4 bits of which indicate the type of packet, as per RFC 1144: 0x40; an unmodified IP datagram (TYPE_IP) 0x70; an uncompressed-TCP/IP datagram (UNCOMPRESSED_TCP), with that byte being the first byte of the raw IP header on the wire, containing the connection number in the protocol field 0x80; a compressed-TCP/IP datagram (COMPRESSED_TCP), with that byte being the first byte of the compressed TCP/IP datagram header for UNCOMPRESSED_TCP, the rest of the modified IP header, and for COMPRESSED_TCP, the compressed TCP/IP datagram header for a total of 16 bytes; the uncompressed IP datagram follows the header. DLT_PPP PPP; if the first 2 bytes are 0xff and 0x03, it's PPP in HDLC-like framing, with the PPP header following those two bytes, otherwise it's PPP without framing, and the packet begins with the PPP header. DLT_FDDI FDDI DLT_ATM_RFC1483 RFC 1483 LLC/SNAP-encapsulated ATM; the packet begins with an IEEE 802.2 LLC header. DLT_RAW Raw IP; the packet begins with an IP header. DLT_PPP_SERIAL PPP in HDLC-like framing, as per RFC 1662, or Cisco PPP with HDLC framing, as per section 4.3.1 of RFC 1547; the first byte will be 0xFF for PPP in HDLC-like framing, and will be 0x0F or 0x8F for Cisco PPP with HDLC framing. DLT_PPP_ETHER PPPoE; the packet begins with a PPPoE header, as per RFC 2516. DLT_C_HDLC Cisco PPP with HDLC framing, as per section 4.3.1 of RFC 1547. DLT_IEEE802_11 IEEE 802.11 wireless LAN. DLT_LOOP OpenBSD loopback encapsulation; the link layer header is a 4-byte field, in network byte order, containing a PF_ value from OpenBSD's socket.h for the network-layer protocol of the packet. Note that, if a savefile is being read, those PF_ values are not necessarily those of the machine reading the capture file. DLT_LINUX_SLL Linux cooked capture encapsulation; the link layer header contains, in order: a 2-byte "packet type", in network byte order, which is one of: 0; packet was sent to us by somebody else. 1; packet was broadcast by somebody else. 2; packet was multicast, but not broadcast, by somebody else. 3; packet was sent by somebody else to somebody else. 4; packet was sent by us. a 2-byte field, in network byte order, containing a Linux ARPHRD_ value for the link layer device type. a 2-byte field, in network byte order, containing the length of the link layer address of the sender of the packet (which could be 0). an 8-byte field containing that number of bytes of the link layer header (if there are more than 8 bytes, only the first 8 are present). a 2-byte field containing an Ethernet protocol type, in network byte order, or containing 1 for Novell 802.3 frames without an 802.2 LLC header or 4 for frames beginning with an 802.2 LLC header. DLT_LTALK Apple LocalTalk; the packet begins with an AppleTalk LLAP header. getnonblock / setnonblock Manipulate the non-blocking flag int getnonblock setnonblock int state DESCRIPTION getnonblock returns the current non-blocking state of the capture descriptor; it always returns 0 on savefiles. DESCRIPTION setnonblock puts a capture descriptor, opened with open_live, into non-blocking mode, or takes it out of non-blocking mode, depending on whether the state argument is non-zero or zero. It has no effect on savefiles. In non-blocking mode, an attempt to read from the capture descriptor with dispatch will, if no packets are currently available to be read, return 0 immediately rather than blocking waiting for packets to arrive. loop and next will not work in non-blocking mode. dump_open Create a Dumper object Dumper dump_open string filename DESCRIPTION dump_open is called to open a savefile for writing and associate it to a newly created Dumper instance. The name - is a synonym for stdout. filename specifies the name of the file to open. close Close a Reader Reader close DESCRIPTION close closes a Reader using pcap_close. Dumper Object Reference dump Dump a packet to a savefile dump Pkthdr header string data DESCRIPTION dump outputs a packet to the savefile opened with dump_open from type Reader. close Close a Dumper Dumper close DESCRIPTION close closes a Dumper. Pkthdr Object Reference getts getcaplen getlen Obtain packet header information (long, long) getts long getcaplen long getlen DESCRIPTION getts, getcaplen and getlen return the timestamp, capture length and total length fields of the packet header, respectively. Timestamp is a tuple with two elements: the number of seconds since the Epoch, and the amount of microseconds past the current second. The capture length is the number of bytes of the packet that are available from the capture. Finally, total length gives the length of the packet, in bytes (which might be more than the number of bytes available from the capture, if the length of the packet is larger than the maximum number of bytes to capture). Bpf Object Reference filter Test a packet against a compiled filter int filter string packet DESCRIPTION filter tests a packet against a compiled filter as returned by pcapy's compile. If the packet is allowed to pass through -1 is returned, otherwise filter returns 0. Bibliography Sources Portions of this work based on pcap 3 by the Lawrence Berkeley National Laboratory, University of California, Berkeley, CA. pcapy-ng-2.1.0/pyproject.toml000066400000000000000000000033131524067405300161520ustar00rootroot00000000000000[build-system] requires = ["setuptools>=61", "wheel"] build-backend = "setuptools.build_meta" # Project metadata is kept in setup.py (it also defines the C Extension); this file adds the # modern build-system declaration and the cibuildwheel configuration for producing # self-contained, precompiled wheels (no deploy-time C compiler or libpcap-dev required). [tool.cibuildwheel] # CPython only (the extension uses the CPython C-API); covers the maintained 3.x series, which # is also exactly what python_requires in setup.py allows and what CI tests. # Free-threaded builds (cp3XXt-*) are deliberately not selected: the module does not declare # Py_mod_gil support yet. build = "cp39-* cp310-* cp311-* cp312-* cp313-* cp314-*" # 32-bit and non-CPython interpreters are not targeted. skip = "*_i686 *-musllinux_i686 pp* gp*" build-verbosity = 1 # Test every built wheel in an isolated env: import it, run the classic loop(), confirm the # fast-path API is present, then run the full fast-path unit test battery against the wheel. test-command = [ "python {project}/tests/wheel_smoketest.py {project}/tests/96pings.pcap", "python -m unittest discover -s {project}/tests -p test_fast.py -v", ] [tool.cibuildwheel.linux] # libpcap headers are needed to COMPILE; auditwheel then vendors libpcap.so INTO the wheel # so the installed wheel is self-contained at runtime. Cover both manylinux (yum/dnf) and # musllinux (apk) base images. before-all = [ "yum install -y libpcap-devel || dnf install -y libpcap-devel || true", "apk add --no-cache libpcap-dev || true", ] [tool.cibuildwheel.macos] # macOS ships libpcap in the SDK; the brew package is only a fallback for older images. before-all = "brew install libpcap || true" pcapy-ng-2.1.0/setup.py000077500000000000000000000111411524067405300147510ustar00rootroot00000000000000import sys import os import glob from setuptools import setup, Extension PACKAGE_NAME = 'pcapy-ng' # You might want to change these to reflect your specific configuration include_dirs = [] library_dirs = [] libraries = [] if sys.platform == 'win32': if os.environ.get('WPDPACK_BASE'): wpdpack = os.environ['WPDPACK_BASE'] include_dirs.append(os.path.join(wpdpack, 'Include')) if sys.maxsize > 2**32: # x64 Python interpreter library_dirs.append(os.path.join(wpdpack, 'Lib', 'x64')) else: # x86 Python interpreter library_dirs.append(os.path.join(wpdpack, 'Lib')) else: # WinPcap include files include_dirs.append(r'c:\wpdpack\Include') # WinPcap library files if sys.maxsize > 2**32: # x64 Python interpreter library_dirs.append(r'c:\wpdpack\Lib\x64') else: # x86 Python interpreter library_dirs.append(r'c:\wpdpack\Lib') libraries = ['wpcap', 'packet', 'ws2_32'] else: libraries = ['pcap'] # end of user configurable parameters macros = [] sources = ['pcapdumper.cc', 'bpfobj.cc', 'pcapobj.cc', 'pcap_pkthdr.cc', 'pcapy.cc' ] if sys.platform == 'win32': sources.append(os.path.join('win32', 'dllmain.cc')) macros.append(('WIN32', '1')) def test_data_files(): # Plain files only: a stale tests/__pycache__ directory (left behind by any earlier test # run in the source tree) makes install_data abort with "can't copy ...: not a regular file". return sorted(f for f in glob.glob(os.path.join('tests', '*')) if os.path.isfile(f) and not f.endswith(('.pyc', '.pyo'))) def read(fname): f = open(os.path.join(os.path.dirname(__file__), fname)) try: return f.read() finally: f.close() setup(name=PACKAGE_NAME, version="2.1.0", url="https://github.com/stamparm/pcapy-ng/", project_urls={ "Source": "https://github.com/stamparm/pcapy-ng/", "Issues": "https://github.com/stamparm/pcapy-ng/issues", "Changelog": "https://github.com/stamparm/pcapy-ng/blob/master/ChangeLog", }, author="Miroslav Stampar", author_email="contact@sekuripy.hr", maintainer="Miroslav Stampar", maintainer_email="contact@sekuripy.hr", platforms=["Linux", "macOS", "Windows"], description="Maintained libpcap binding for Python (Pcapy-compatible) with optional in-C packet classification", long_description=read('README.md'), long_description_content_type="text/markdown", keywords=["pcap", "libpcap", "sniffer", "packet capture", "network", "pcapy", "npcap"], license="Apache-2.0", # CPython 3.9-3.14: what CI builds and tests, and what wheels are published for. The C # sources still carry their Python 2 paths, so `python setup.py install` under 2.7 can # still work, but nothing tests it and pip metadata does not claim it. python_requires=">=3.9", classifiers=[ "Development Status :: 5 - Production/Stable", "Intended Audience :: Developers", "Intended Audience :: System Administrators", "Intended Audience :: Telecommunications Industry", # NOTE: no "License ::" classifier on purpose -- superseded by the SPDX # expression in license= above (setuptools >= 77 deprecates the classifiers). "Operating System :: MacOS :: MacOS X", "Operating System :: Microsoft :: Windows", "Operating System :: POSIX :: Linux", "Operating System :: POSIX :: BSD", "Programming Language :: C++", "Programming Language :: Python :: 3", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", "Programming Language :: Python :: 3.13", "Programming Language :: Python :: 3.14", "Programming Language :: Python :: Implementation :: CPython", "Topic :: Security", "Topic :: System :: Networking :: Monitoring", ], ext_modules=[Extension( name="pcapy", sources=sources, define_macros=macros, include_dirs=include_dirs, library_dirs=library_dirs, libraries=libraries)], #scripts=['tests/pcapytests.py', 'tests/96pings.pcap'], data_files=[ (os.path.join('share', 'doc', PACKAGE_NAME), ['README', 'README.md', 'LICENSE', 'pcapy.html']), (os.path.join('share', 'doc', PACKAGE_NAME, 'tests'), test_data_files())] ) pcapy-ng-2.1.0/tests/000077500000000000000000000000001524067405300144005ustar00rootroot00000000000000pcapy-ng-2.1.0/tests/96pings.pcap000077500000000000000000000250301524067405300165470ustar00rootroot00000000000000`Hcؾ`b$+]ET@eQHcT  !"#$%&'()*+,-./012345Hd`b$+]ET@ePHd~  !"#$%&'()*+,-./012345He`b$+]ET@eKHeq  !"#$%&'()*+,-./012345Hf`b$+]ET@eEHf]  !"#$%&'()*+,-./012345Hg`b$+]ET@eDHgy  !"#$%&'()*+,-./012345Hh`b$+]ET@eCHh  !"#$%&'()*+,-./012345Hi`b$+]ET@eBxHi  !"#$%&'()*+,-./012345Hjq`b$+]ET@eTHj  !"#$%&'()*+,-./012345Hkܦ`b$+]ET@eMHkE  !"#$%&'()*+,-./012345Hlݳ`b$+]ET@eL HlT  !"#$%&'()*+,-./012345Hm޻`b$+]ET@eK Hm\  !"#$%&'()*+,-./012345Hn`b$+]ET4@eI Hn|  !"#$%&'()*+,-./012345Ho`b$+]ETW@eHe Ho  !"#$%&'()*+,-./012345Hp`b$+]ET\@eG^ Hp  !"#$%&'()*+,-./012345Hq `b$+]ET]@eFSHq  !"#$%&'()*+,-./012345Hr`b$+]ET^@eDbHr  !"#$%&'()*+,-./012345Hsz`b$+]ET`@eBHs  !"#$%&'()*+,-./012345Ht`b$+]ETb@eAHt(  !"#$%&'()*+,-./012345Hu`b$+]ETc@e@Hu-  !"#$%&'()*+,-./012345Hv`b$+]ETe@e?Hv5  !"#$%&'()*+,-./012345Hw{`b$+]ETf@esHw{{  !"#$%&'()*+,-./012345Hx|`b$+]ETg@elHx|  !"#$%&'()*+,-./012345Hy~`b$+]ETi@eDHy}  !"#$%&'()*+,-./012345Hz~`b$+]ETl@eHz~a  !"#$%&'()*+,-./012345H{`b$+]ETm@eH{[  !"#$%&'()*+,-./012345H| `b$+]ETs@e$H|  !"#$%&'()*+,-./012345H}?`b$+]ETw@{exH}T  !"#$%&'()*+,-./012345H~`b$+]ETx@ze4eH~z  !"#$%&'()*+,-./012345H>`b$+]ETy@ye.H  !"#$%&'()*+,-./012345H`b$+]ETz@xe'2H  !"#$%&'()*+,-./012345H `b$+]ET@se H   !"#$%&'()*+,-./012345H `b$+]ET@qe4H   !"#$%&'()*+,-./012345H`b$+]ET@pe H  !"#$%&'()*+,-./012345H#`b$+]ET@oe!H  !"#$%&'()*+,-./012345HR`b$+]ET@le"H  !"#$%&'()*+,-./012345HO`b$+]ET@ke#H  !"#$%&'()*+,-./012345HV`b$+]ET@je$H  !"#$%&'()*+,-./012345HV`b$+]ET@ge%H  !"#$%&'()*+,-./012345HY`b$+]ET@ee&H  !"#$%&'()*+,-./012345HX`b$+]ET@ce'H  !"#$%&'()*+,-./012345Hd`b$+]ET@be(H  !"#$%&'()*+,-./012345Hg`b$+]ET@_e)H   !"#$%&'()*+,-./012345H_`b$+]ET@\e*H  !"#$%&'()*+,-./012345H[`b$+]ET@Te)+H[  !"#$%&'()*+,-./012345H]`b$+]ET@SeJ,H]r  !"#$%&'()*+,-./012345Hp`b$+]ET@Qei-HpQ  !"#$%&'()*+,-./012345Hq`b$+]ET@OeW.Hqa  !"#$%&'()*+,-./012345Hr`b$+]ET@Ne[/Hr[  !"#$%&'()*+,-./012345Hs`b$+]ET@LeN0Hsf  !"#$%&'()*+,-./012345Ht`b$+]ET@KeA1Htq  !"#$%&'()*+,-./012345H~`b$+]ET@Dey2H~7  !"#$%&'()*+,-./012345HB`b$+]ET@@e3H  !"#$%&'()*+,-./012345Hi`b$+]ET@1e4H  !"#$%&'()*+,-./012345H`b$+]ET@)e5H  !"#$%&'()*+,-./012345H`b$+]ET@ e6H  !"#$%&'()*+,-./012345H`b$+]ET@e7H  !"#$%&'()*+,-./012345H:`b$+]ET @e8H  !"#$%&'()*+,-./012345HL`b$+]ET/@e}9H  !"#$%&'()*+,-./012345Hd`b$+]ET<@e|:H  !"#$%&'()*+,-./012345H`b$+]ET>@ek;H  !"#$%&'()*+,-./012345H"`b$+]ETB@ejHW  !"#$%&'()*+,-./012345HY`b$+]ETZ@e<?HYX  !"#$%&'()*+,-./012345H[d`b$+]ETm@eΌ@H[  !"#$%&'()*+,-./012345H\h`b$+]ETn@e͉AH\  !"#$%&'()*+,-./012345Ha`b$+]ETo@e0BHa^  !"#$%&'()*+,-./012345Hb`b$+]ETy@ye#CHbi  !"#$%&'()*+,-./012345He`b$+]ET@feDHez  !"#$%&'()*+,-./012345Hf`b$+]ET@YeEHf  !"#$%&'()*+,-./012345Ho`b$+]ET@Be<FHoJ  !"#$%&'()*+,-./012345Hp`b$+]ET@7eGHpo  !"#$%&'()*+,-./012345Hq`b$+]ET@1eHHq  !"#$%&'()*+,-./012345Hr`b$+]ET@,eIHrx  !"#$%&'()*+,-./012345Ht`b$+]ET@+eJHta  !"#$%&'()*+,-./012345Hw`b$+]ET@"eKHw  !"#$%&'()*+,-./012345HyQ`b$+]ET@eLHx  !"#$%&'()*+,-./012345H]`b$+]ET@e|MH  !"#$%&'()*+,-./012345H`b$+]ET@ eMNH)  !"#$%&'()*+,-./012345H`b$+]ET@e2OHB  !"#$%&'()*+,-./012345H5`b$+]ET@ePH  !"#$%&'()*+,-./012345H\`b$+]ET@evQH  !"#$%&'()*+,-./012345Hk`b$+]ET@egRH  !"#$%&'()*+,-./012345H`b$+]ET@eSH  !"#$%&'()*+,-./012345H<`b$+]ET@eTH  !"#$%&'()*+,-./012345HI`b$+]ET @eUH  !"#$%&'()*+,-./012345HK`b$+]ET@eyVH  !"#$%&'()*+,-./012345H`b$+]ET@eWH  !"#$%&'()*+,-./012345H`b$+]ET!@eXH  !"#$%&'()*+,-./012345H6`b$+]ET.@eYH  !"#$%&'()*+,-./012345HU`b$+]ET8@emZH  !"#$%&'()*+,-./012345H``b$+]ET:@eZ[H  !"#$%&'()*+,-./012345H^`b$+]ET=@e\\H  !"#$%&'()*+,-./012345Hk`b$+]ET?@ex"]H6  !"#$%&'()*+,-./012345HD`b$+]ETB@ewr^H  !"#$%&'()*+,-./012345H>`b$+]ETC@evs_H  !"#$%&'()*+,-./012345pcapy-ng-2.1.0/tests/_pcapgen.py000066400000000000000000000123251524067405300165310ustar00rootroot00000000000000# coding: utf-8 """Test helper: build pcap files (and raw packets) for the pcapy-ng fast-path battery. stdlib only; Py2/3.""" import os import struct import socket import tempfile DLT_EN10MB = 1 DLT_RAW = 12 # raw IP, no link-layer header (IP starts at offset 0) DLT_LINUX_SLL = 113 # Linux "cooked" capture (the 'any' interface): 16-byte header PCAP_MAGIC = 0xA1B2C3D4 def eth_vlan(vid=100, inner=0x0800, tags=1): # Ethernet header with one or more 802.1Q VLAN tags; IP follows at offset 14 + 4*tags. out = b"\xaa\xbb\xcc\xdd\xee\xff\x11\x22\x33\x44\x55\x66" # dst+src for _ in range(tags): out += b"\x81\x00" + struct.pack("!H", vid) # TPID + TCI per tag out += struct.pack("!H", inner) # final ethertype (0x0800) return out def sll(ethertype=0x0800): # 16-byte LINUX_SLL cooked header: pkttype(2) arphrd(2) addrlen(2) addr(8) proto(2) return struct.pack("!HHH", 0, 1, 6) + b"\x11\x22\x33\x44\x55\x66\x00\x00" + struct.pack("!H", ethertype) BAD_IPS = ["45.95.202.106", "2.200.107.168"] BAD_DOMS = ["j0mla.sytes.net", "atacoinc8897.hopto.org"] def eth(ethertype=0x0800): return b"\xaa\xbb\xcc\xdd\xee\xff\x11\x22\x33\x44\x55\x66" + struct.pack("!H", ethertype) def ipv4(proto, src, dst, payload, ihl=5, total_len=None): vihl = 0x40 | (ihl & 0x0f) tot = total_len if total_len is not None else (ihl * 4 + len(payload)) return struct.pack("!BBHHHBBH4s4s", vihl, 0, tot, 0x1234, 0, 64, proto, 0, socket.inet_aton(src), socket.inet_aton(dst)) + payload def ipv6(proto, src, dst, payload): # minimal IPv6 header (40 bytes), ethertype should be 0x86DD return (struct.pack("!IHBB", 0x60000000, len(payload), proto, 64) + socket.inet_pton(socket.AF_INET6, src) + socket.inet_pton(socket.AF_INET6, dst) + payload) def udp(sport, dport, payload): return struct.pack("!HHHH", sport, dport, 8 + len(payload), 0) + payload def tcp(sport, dport, flags, payload, doff=5): off = (doff & 0x0f) << 4 return struct.pack("!HHIIBBHHH", sport, dport, 1, 1, off, flags, 65535, 0, 0) + payload def dns_query(name): q = b"" for lbl in name.split("."): q += struct.pack("B", len(lbl)) + lbl.encode() q += b"\x00\x00\x01\x00\x01" return struct.pack("!HHHHHH", 0x1234, 0x0100, 1, 0, 0, 0) + q def tls_client_hello(sni, with_record=True): s = sni.encode() srv = b"\x00" + struct.pack("!H", len(s)) + s lst = struct.pack("!H", len(srv)) + srv ext = b"\x00\x00" + struct.pack("!H", len(lst)) + lst body = (b"\x03\x03" + b"\x11" * 32 + b"\x00" + b"\x00\x02\x13\x01" + b"\x01\x00" + struct.pack("!H", len(ext)) + ext) hs = b"\x01" + struct.pack("!I", len(body))[1:] + body if with_record: return b"\x16\x03\x03" + struct.pack("!H", len(hs)) + hs return hs # ---- convenience full-frame builders (Ethernet/IPv4) ---- def dns_packet(name, client="10.0.0.5", server="8.8.8.8", sport=12345): return eth() + ipv4(17, client, server, udp(sport, 53, dns_query(name))) def tls_packet(sni, client="10.0.0.5", server="93.184.216.34", sport=40000, psh=True): flags = 0x18 if psh else 0x10 return eth() + ipv4(6, client, server, tcp(sport, 443, flags, tls_client_hello(sni))) def http_packet(host="example.com", client="10.0.0.5", server="93.184.216.34", sport=40001): req = b"GET / HTTP/1.1\r\nHost: " + host.encode() + b"\r\n\r\n" return eth() + ipv4(6, client, server, tcp(sport, 80, 0x18, req)) def quicish_packet(client="10.0.0.5", server="93.184.216.34", sport=40002, payload_len=1100): return eth() + ipv4(17, client, server, udp(sport, 443, b"\x00" * payload_len)) def ioc_packet(bad_ip, client="10.0.0.5", sport=40003, dport=4444): return eth() + ipv4(17, client, bad_ip, udp(sport, dport, b"\x00" * 32)) def icmp_packet(dst, client="10.0.0.5", typ=8): return eth() + ipv4(1, client, dst, struct.pack("!BBHHH", typ, 0, 0, 0, 1) + b"payload") def syn_packet(dst="1.2.3.4", client="10.0.0.5", sport=40000, dport=12345): return eth() + ipv4(6, client, dst, tcp(sport, dport, 0x02, b"")) # pure SYN (flags==2) def write_pcap(path, packets, linktype=DLT_EN10MB, snaplen=65535, ts=1700000000): with open(path, "wb") as f: f.write(struct.pack("!IHHIIII", PCAP_MAGIC, 2, 4, 0, 0, snaplen, linktype)) for i, p in enumerate(packets): f.write(struct.pack("!IIII", ts, i % 1000000, len(p), len(p)) + p) return path def temp_pcap(packets, **kw): fd, path = tempfile.mkstemp(suffix=".pcap") os.close(fd) return write_pcap(path, packets, **kw) def write_pcap_trunc(path, records, linktype=DLT_EN10MB, snaplen=65535, ts=1700000000): """records = list of (wire_len, captured_bytes); writes incl_len=len(captured_bytes) but orig_len=wire_len, i.e. a snaplen-truncated capture.""" with open(path, "wb") as f: f.write(struct.pack("!IHHIIII", PCAP_MAGIC, 2, 4, 0, 0, snaplen, linktype)) for i, (wire_len, data) in enumerate(records): f.write(struct.pack("!IIII", ts, i % 1000000, len(data), wire_len) + data) return path def temp_pcap_trunc(records, **kw): fd, path = tempfile.mkstemp(suffix=".pcap") os.close(fd) return write_pcap_trunc(path, records, **kw) pcapy-ng-2.1.0/tests/mem_leak_test.py000066400000000000000000000054571524067405300175760ustar00rootroot00000000000000# coding: utf-8 """Memory-leak hunt for the pcapy-ng C fast path. Runs loop_filtered / loop_to_buffer thousands of times and asserts resident memory stays flat -- catches leaks in ipset_build / flowtab_build (12 MB/call!) / port arrays / per-packet header objects. RAM-SAFE: intended to run under `ulimit -v`. A real per-call leak would blow the cap (=> killed/MemoryError = test fails loudly) rather than the host. Exit 0 = stable, 1 = leak/regression. Py2 + Py3.""" import os import sys import gc import socket sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import pcapy import _pcapgen as G IOCSET = b"".join(socket.inet_aton(x) for x in G.BAD_IPS) DPI = [80, 8080, 3128, 8000, 8118, 1080] TLS = [443] PAGE_KB = os.sysconf("SC_PAGE_SIZE") // 1024 if hasattr(os, "sysconf") else 4 def rss_kb(): try: with open("/proc/self/statm") as f: return int(f.read().split()[1]) * PAGE_KB except Exception: import resource return resource.getrusage(resource.RUSAGE_SELF).ru_maxrss def make_pcap(): pkts = [] for i in range(1000): pkts.append(G.dns_packet("h%d.example.com" % (i % 50))) # admitted (DNS) -> exercises hdr alloc pkts.append(G.quicish_packet(sport=40000 + (i % 500))) # noise / flow-head churn if i % 10 == 0: pkts.append(G.ioc_packet(G.BAD_IPS[i % 2])) pkts.append(G.http_packet()) return G.temp_pcap(pkts) def run_filtered(path, iters): for _ in range(iters): r = pcapy.open_offline(path) r.loop_filtered(-1, lambda h, d, c: None, 0xFF, IOCSET, 2, DPI, TLS, 53) # ioc+flowtab+ports each call r.close() def run_buffer(path, iters): buf = bytearray(1 << 22) for _ in range(iters): r = pcapy.open_offline(path) r.loop_to_buffer(-1, buf, 0xFF, IOCSET, 2, DPI, TLS, 53) r.close() def measure(name, fn, path, warm, hot, budget_kb): fn(path, warm) gc.collect() base = rss_kb() fn(path, hot) gc.collect() after = rss_kb() delta = after - base ok = delta <= budget_kb print(" %-18s warm=%d hot=%d RSS %d->%d KB delta=%+d KB budget=%d %s" % (name, warm, hot, base, after, delta, budget_kb, "OK" if ok else "LEAK?!")) return ok def main(): path = make_pcap() try: ok = True # flowtab is 12 MB/call: if leaked, 400 calls => ~4.8 GB (blows ulimit). Stable => tiny delta. ok &= measure("loop_filtered", run_filtered, path, 40, 400, 40000) ok &= measure("loop_to_buffer", run_buffer, path, 40, 400, 40000) print("RESULT:", "STABLE (no leak)" if ok else "LEAK DETECTED") return 0 if ok else 1 finally: os.unlink(path) if __name__ == "__main__": sys.exit(main()) pcapy-ng-2.1.0/tests/pcapytests.py000077500000000000000000000110721524067405300171550ustar00rootroot00000000000000## Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) ## Copyright (c) 2014 CORE Security Technologies ## ## This software is provided under under the Apache Software License. ## See the accompanying LICENSE file for more information. ## import pcapy import sys import unittest import os class TestPcapy(unittest.TestCase): _96PINGS = '96pings.pcap' _IFACE = 'vboxnet0' def testPacketHeaderRefCount(self): """ #1: when next() creates a pkthdr it makes one extra reference """ class _Simple: pass r = pcapy.open_offline(TestPcapy._96PINGS) # get one & check its refcount self.assertEqual( sys.getrefcount(r.next()[0]), sys.getrefcount(_Simple())) def testEOFValue(self): """ #2 empty string is returned as packet body at end of file """ r = pcapy.open_offline(TestPcapy._96PINGS) # get one & check its refcount i = 0 refNone = sys.getrefcount(None) hdr, pkt = r.next() while hdr is not None: hdr, pkt = r.next() i += 1 self.assertEqual(96, i) self.assertTrue(hdr is None) self.assertEqual(pkt, b'') del hdr self.assertEqual(refNone, sys.getrefcount(None)) def testBPFFilter(self): """ #3 test offline BPFFilter """ r = pcapy.open_offline(TestPcapy._96PINGS) bpf = pcapy.BPFProgram("ip dst host 192.168.1.1") hdr, pkt = r.next() while hdr is not None: f = bpf.filter(pkt) self.assertNotEqual(f, 0) hdr, pkt = r.next() def _testLiveCapture(self): """ #4 (disabled -- requires interface info) test live capture """ r = pcapy.open_live(TestPcapy._IFACE, 60000, 1, 1500) net = r.getnet() self.assertEqual(net, '192.168.56.0') hdr, body = r.next() self.assertTrue(hdr is not None) def _testSendPacket(self): """ #5 (disabled -- requires interface info) test sendpacket """ r = pcapy.open_offline(TestPcapy._96PINGS) w = pcapy.open_live(TestPcapy._IFACE, 60000, 1, 1500) # get one & check its refcount i = 0 hdr, pkt = r.next() while hdr is not None: w.sendpacket(pkt) hdr, pkt = r.next() i += 1 def testPacketDumper(self): """ #6 test that the dumper writes correct payload """ try: r = pcapy.open_offline(TestPcapy._96PINGS) dumper = r.dump_open('tmp.pcap') hdr, body = r.next() i = 0 while hdr is not None: dumper.dump(hdr, body) i += 1 hdr, body = r.next() # make sure file closes del dumper # check that the dumper wrote a legal pcap # file with same packer data r = pcapy.open_offline(TestPcapy._96PINGS) r2 = pcapy.open_offline('tmp.pcap') h1, b1 = r.next() h2, b2 = r2.next() while h1 is not None and h2 is not None: self.assertEqual(b1, b2) h1, b1 = r.next() h2, b2 = r2.next() self.assertTrue(h1 is None) self.assertTrue(h2 is None) del r2 finally: os.unlink('tmp.pcap') def testClose(self): """ #7 Test the close method """ r = pcapy.open_offline(TestPcapy._96PINGS) hdr, body = r.next() assert hdr is not None r.close() with self.assertRaises(ValueError): r.next() def testContextManager(self): """ #8 Test the context manager support """ with pcapy.open_offline(TestPcapy._96PINGS) as r: hdr, body = r.next() assert hdr is not None with self.assertRaises(ValueError): r.next() def test_get_bpf(self): bpf = pcapy.compile(pcapy.DLT_EN10MB, 2**16, "icmp", 1, 1) code = bpf.get_bpf() # result of `tcpdump "icmp" -ddd -s 65536` on EN10MB interface expected = """6 40 0 0 12 21 0 3 2048 48 0 0 23 21 0 1 1 6 0 0 65536 6 0 0 0""" result = str(len(code)) + "\n" result += "\n".join([' '.join(map(str, inst)) for inst in code]) self.assertEqual(expected, result) suite = unittest.TestLoader().loadTestsFromTestCase(TestPcapy) result = unittest.TextTestRunner(verbosity=2).run(suite) if not result.wasSuccessful(): sys.exit(1) pcapy-ng-2.1.0/tests/test_fast.py000066400000000000000000001161571524067405300167610ustar00rootroot00000000000000# coding: utf-8 """Battery of unit tests for the pcapy-ng fast path (loop_filtered, loop_to_buffer, filtered_stats, next_batch, configurable classifier, IOC set, flow-cutoff) plus the classic API and adversarial robustness. stdlib unittest; runs on Py2 and Py3. Run: python3 -m unittest -v test_fast (and the same under python2) """ import os import sys import struct import socket import random import threading import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import pcapy import _pcapgen as G # result-tuple indices: (admitted, dropped, dns, dpi, noise, ioc, head, syn) ADM, DROP, DNS, DPI, NOISE, IOC, HEAD, SYN = range(8) # class indices (what the classifier returns / admit_mask bits select) C_DNS, C_DPI, C_NOISE, C_IOC, C_HEAD, C_SYN = range(6) # admit-mask bit indices P_DNS, P_DPI, P_NOISE, P_IOC, P_HEAD, P_SYN = range(100, 106) # public cls ids handed to the callback ADMIT_ALL = 0xFF IOCSET = b"".join(socket.inet_aton(x) for x in G.BAD_IPS) def run_filtered(packets, admit_mask=ADMIT_ALL, ioc=IOCSET, cutoff=0, dpi=None, tls=None, dns=53, collect=False): path = G.temp_pcap(packets) try: r = pcapy.open_offline(path) got = [] def cb(hdr, data, cls): if collect: got.append((cls, bytes(data))) return None res = r.loop_filtered(-1, cb, admit_mask, ioc, cutoff, dpi, tls, dns, 14, 1) # profile=1 return res, got finally: os.unlink(path) class TestClassification(unittest.TestCase): def test_dns_classified(self): res, _ = run_filtered([G.dns_packet("example.com")] * 5) self.assertEqual(res[DNS], 5) def test_dpi_psh_on_http_port(self): res, _ = run_filtered([G.http_packet()] * 3) self.assertEqual(res[DPI], 3) def test_non_psh_http_payload_is_dpi(self): # Maltrail inspects HTTP responses too (not always PSH) -> any payload-bearing TCP # packet on a watched port is DPI, even without the PSH flag. pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(40000, 80, 0x10, b"x" * 10)) res, _ = run_filtered([pkt]) self.assertEqual(res[DPI], 1) self.assertEqual(res[NOISE], 0) def test_bare_ack_on_http_port_is_noise(self): # no payload -> carries no HTTP -> noise (and IOC would catch it if to a trail IP) pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(40000, 80, 0x10, b"")) res, _ = run_filtered([pkt]) self.assertEqual(res[NOISE], 1) self.assertEqual(res[DPI], 0) def test_http_port_off_port_payload_not_dpi(self): # payload-bearing TCP on a NON-watched port is not DPI (default capture filter excludes it) pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(40000, 12345, 0x18, b"x" * 10)) res, _ = run_filtered([pkt]) self.assertEqual(res[DPI], 0) self.assertEqual(res[NOISE], 1) def test_quic_bulk_is_noise_without_cutoff(self): res, _ = run_filtered([G.quicish_packet()] * 4, cutoff=0) self.assertEqual(res[NOISE], 4) self.assertEqual(res[HEAD], 0) def test_ioc_ip_kept_even_as_udp_noise(self): res, _ = run_filtered([G.ioc_packet(G.BAD_IPS[0]), G.ioc_packet(G.BAD_IPS[1])]) self.assertEqual(res[IOC], 2) self.assertEqual(res[NOISE], 0) def test_ioc_takes_priority_over_flowhead(self): # bad IP on tcp/443 PSH -> still IOC (3), not HEAD(4) pkt = G.eth() + G.ipv4(6, "10.0.0.5", G.BAD_IPS[0], G.tcp(5000, 443, 0x10, b"x" * 20)) res, _ = run_filtered([pkt], cutoff=2) self.assertEqual(res[IOC], 1) self.assertEqual(res[HEAD], 0) def test_dns_over_tcp(self): pkt = G.eth() + G.ipv4(6, "10.0.0.5", "8.8.8.8", G.tcp(5000, 53, 0x18, b"\x00" * 20)) res, _ = run_filtered([pkt]) self.assertEqual(res[DNS], 1) def test_ipv6_is_noise(self): pkt = G.eth(0x86DD) + G.ipv6(17, "::1", "::2", G.udp(53, 53, b"\x00" * 8)) res, _ = run_filtered([pkt]) # classifier is IPv4-only -> treated as noise (and must not crash) self.assertEqual(res[ADM] + res[DROP], 1) def test_arp_is_noise(self): pkt = G.eth(0x0806) + b"\x00" * 28 res, _ = run_filtered([pkt]) self.assertEqual(res[NOISE], 1) class TestAdmitMask(unittest.TestCase): def setUp(self): self.pkts = [G.dns_packet("a.com"), G.http_packet(), G.quicish_packet(), G.ioc_packet(G.BAD_IPS[0]), G.tls_packet("x.com")] def test_admit_none(self): res, got = run_filtered(self.pkts, admit_mask=0, cutoff=2, collect=True) self.assertEqual(res[ADM], 0) self.assertEqual(len(got), 0) self.assertEqual(res[DROP], len(self.pkts)) def test_admit_only_dns(self): res, got = run_filtered(self.pkts, admit_mask=(1 << C_DNS), cutoff=2, collect=True) self.assertEqual(res[ADM], 1) self.assertTrue(all(c == P_DNS for c, _ in got)) def test_admit_all_default_mask_drops_noise(self): # default 27 = DNS|DPI|IOC|HEAD (no noise) res, _ = run_filtered(self.pkts, admit_mask=27, cutoff=2) self.assertEqual(res[ADM], res[DNS] + res[DPI] + res[IOC] + res[HEAD]) def test_classification_independent_of_mask(self): a, _ = run_filtered(self.pkts, admit_mask=0, cutoff=2) b, _ = run_filtered(self.pkts, admit_mask=ADMIT_ALL, cutoff=2) self.assertEqual(a[DNS:], b[DNS:]) # class counts identical regardless of admit class TestConfigurablePorts(unittest.TestCase): def test_custom_dns_port(self): pkt = G.eth() + G.ipv4(17, "10.0.0.5", "8.8.8.8", G.udp(5353, 5353, b"\x00" * 10)) res, _ = run_filtered([pkt], dns=5353) self.assertEqual(res[DNS], 1) res2, _ = run_filtered([pkt], dns=53) self.assertEqual(res2[DNS], 0) def test_custom_dpi_ports(self): pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(5000, 8443, 0x18, b"x" * 10)) res, _ = run_filtered([pkt], dpi=[8443]) self.assertEqual(res[DPI], 1) res2, _ = run_filtered([pkt]) # default ports don't include 8443 self.assertEqual(res2[DPI], 0) def test_custom_tls_ports_for_cutoff(self): pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(5000, 8443, 0x10, b"x" * 10)) res, _ = run_filtered([pkt], cutoff=2, tls=[8443]) self.assertEqual(res[HEAD], 1) def test_ports_as_tuple(self): pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(5000, 9999, 0x18, b"x" * 10)) res, _ = run_filtered([pkt], dpi=(9999,)) self.assertEqual(res[DPI], 1) def test_empty_port_lists(self): # empty dpi list -> nothing is DPI res, _ = run_filtered([G.http_packet()], dpi=[]) self.assertEqual(res[DPI], 0) def test_non_int_ports_raise(self): path = G.temp_pcap([G.dns_packet("a.com")]) try: r = pcapy.open_offline(path) self.assertRaises(Exception, r.loop_filtered, -1, lambda h, d, c: None, 27, IOCSET, 0, ["nope"], None, 53) finally: os.unlink(path) class TestIOCSet(unittest.TestCase): def test_empty_ioc(self): res, _ = run_filtered([G.ioc_packet(G.BAD_IPS[0])], ioc=b"") self.assertEqual(res[IOC], 0) self.assertEqual(res[NOISE], 1) def test_ioc_under_4_bytes_ignored(self): res, _ = run_filtered([G.ioc_packet(G.BAD_IPS[0])], ioc=b"\x01\x02\x03") self.assertEqual(res[IOC], 0) def test_ioc_src_or_dst_match(self): # bad IP as SOURCE pkt = G.eth() + G.ipv4(17, G.BAD_IPS[0], "10.0.0.5", G.udp(4444, 5000, b"\x00" * 8)) res, _ = run_filtered([pkt]) self.assertEqual(res[IOC], 1) def test_many_iocs_hash_set(self): ips = ["1.2.3.%d" % i for i in range(1, 200)] ioc = b"".join(socket.inet_aton(x) for x in ips) pkt = G.eth() + G.ipv4(17, "10.0.0.5", "1.2.3.150", G.udp(5000, 6000, b"\x00" * 8)) res, _ = run_filtered([pkt], ioc=ioc) self.assertEqual(res[IOC], 1) class TestFlowCutoff(unittest.TestCase): def _flow(self, n): sport = 50000 return [G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(sport, 443, 0x10, b"\x00" * 50)) for _ in range(n)] def test_cutoff_keeps_first_n(self): res, _ = run_filtered(self._flow(10), cutoff=2) self.assertEqual(res[HEAD], 2) self.assertEqual(res[NOISE], 8) def test_cutoff_zero_disables(self): res, _ = run_filtered(self._flow(10), cutoff=0) self.assertEqual(res[HEAD], 0) self.assertEqual(res[NOISE], 10) def test_cutoff_one(self): res, _ = run_filtered(self._flow(5), cutoff=1) self.assertEqual(res[HEAD], 1) class TestFilteredStats(unittest.TestCase): def test_stats_match_return(self): pkts = [G.dns_packet("a.com"), G.http_packet(), G.ioc_packet(G.BAD_IPS[0])] path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) self.assertEqual(tuple(r.filtered_stats()), tuple(res)) finally: os.unlink(path) def test_stats_reset_between_runs(self): path = G.temp_pcap([G.dns_packet("a.com")] * 4) try: r = pcapy.open_offline(path) r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) r2 = pcapy.open_offline(path) r2.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) self.assertEqual(r2.filtered_stats()[DNS], 4) # not 8 finally: os.unlink(path) def test_stats_live_from_other_thread(self): pkts = [G.dns_packet("a.com")] * 2000 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) samples = [] def busy(h, d, c): # touch data so the loop takes measurable time _ = sum(bytearray(d[:4])) return None def run(): r.loop_filtered(-1, busy, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) t = threading.Thread(target=run) t.start() while t.is_alive(): samples.append(r.filtered_stats()[ADM]) t.join() final = r.filtered_stats() self.assertEqual(final[DNS], 2000) # counters never decreased self.assertEqual(samples, sorted(samples)) finally: os.unlink(path) class TestLoopToBuffer(unittest.TestCase): def _decode(self, buf, n): out = [] off = 0 mv = bytearray(buf) for _ in range(n): ln = mv[off] | (mv[off+1] << 8) | (mv[off+2] << 16) | (mv[off+3] << 24) cls = mv[off+4] out.append((cls, bytes(mv[off+5:off+5+ln]))) off += 5 + ln return out, off def test_writes_admitted_only(self): pkts = [G.dns_packet("a.com"), G.http_packet(), G.quicish_packet()] path = G.temp_pcap(pkts) try: buf = bytearray(1 << 20) r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, 27, IOCSET, 0, None, None, 53, 14, 1) # cutoff 0 -> quic stays noise, dropped written, dropped, overflow, used = res[:4] self.assertEqual(written, 2) # dns + dpi; quic noise dropped self.assertEqual(overflow, 0) decoded, consumed = self._decode(buf, written) self.assertEqual(consumed, used) self.assertEqual(set(c for c, _ in decoded), set([P_DNS, P_DPI])) finally: os.unlink(path) def test_matches_loop_filtered_counts(self): pkts = ([G.dns_packet("a.com")] * 10 + [G.http_packet()] * 5 + [G.quicish_packet()] * 20 + [G.ioc_packet(G.BAD_IPS[0])] * 3) path = G.temp_pcap(pkts) try: f = pcapy.open_offline(path).loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) buf = bytearray(1 << 21) b = pcapy.open_offline(path).loop_to_buffer(-1, buf, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) self.assertEqual(b[4:], f[2:]) # dns,dpi,noise,ioc,head identical finally: os.unlink(path) def test_overflow_is_safe(self): # tiny buffer; many admitted -> overflow>0, used<=cap, no crash/corruption pkts = [G.dns_packet("a.com")] * 1000 path = G.temp_pcap(pkts) try: buf = bytearray(200) # holds only a couple of packets r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) written, dropped, overflow, used = res[:4] self.assertEqual(written + overflow, 1000) self.assertTrue(used <= len(buf)) self.assertTrue(written >= 1) finally: os.unlink(path) class TestNextBatch(unittest.TestCase): def test_reads_all(self): pkts = [G.dns_packet("a.com")] * 137 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) total = 0 while True: buf, meta = r.next_batch(50) if not meta: break total += len(meta) // 16 # packed 16-byte records self.assertEqual(total, 137) finally: os.unlink(path) class TestNextBatchEdge(unittest.TestCase): def test_max_n_zero_and_negative_clamp(self): pkts = [G.dns_packet("a.com")] * 10 for mn in (0, -5): path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) buf, meta = r.next_batch(mn) # clamps internally; must not crash self.assertTrue(len(meta) % 16 == 0) self.assertTrue(len(meta) // 16 >= 1) finally: os.unlink(path) def test_large_max_n_ok(self): pkts = [G.dns_packet("a.com")] * 30 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) total = 0 while True: buf, meta = r.next_batch(10000000) # 10M -> ~160MB meta alloc, fine under cap if not meta: break total += len(meta) // 16 self.assertEqual(total, 30) finally: os.unlink(path) class TestSnaplenTruncation(unittest.TestCase): """caplen < wire length (snaplen-truncated capture): the C path must use caplen and never read past it -> no crash, sane classification.""" def test_truncated_mid_ip(self): full = G.dns_packet("trunc.example.com") recs = [(len(full), full[:20]), # only 20 bytes captured (mid IP header) (len(full), full[:14]), # only L2 captured (len(full), full[:36]), # truncated BEFORE the UDP dest-port field (len(full), full)] # complete path = G.temp_pcap_trunc(recs) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) self.assertEqual(res[ADM] + res[DROP], 4) # all accounted, no crash self.assertEqual(res[DNS], 1) # caplen bound: only the complete one sees port 53 finally: os.unlink(path) def test_truncated_to_buffer(self): full = G.dns_packet("a.com") recs = [(len(full), full[:n]) for n in range(0, len(full), 3)] + [(len(full), full)] path = G.temp_pcap_trunc(recs) try: buf = bytearray(1 << 16) r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) self.assertEqual(res[0] + res[1] + res[2], len(recs)) # written+dropped+overflow self.assertTrue(res[3] <= len(buf)) finally: os.unlink(path) class TestBackCompat(unittest.TestCase): def test_three_arg_loop_filtered(self): path = G.temp_pcap([G.dns_packet("a.com")] * 3) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL) self.assertEqual(res[DNS], 3) finally: os.unlink(path) def test_five_arg_loop_filtered(self): path = G.temp_pcap([G.ioc_packet(G.BAD_IPS[0])]) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, 27, IOCSET, 2, None, None, 53, 14, 1) self.assertEqual(res[IOC], 1) finally: os.unlink(path) class TestErrorHandling(unittest.TestCase): def test_callback_exception_propagates(self): path = G.temp_pcap([G.dns_packet("a.com")] * 5) try: r = pcapy.open_offline(path) def boom(h, d, c): raise ValueError("boom") self.assertRaises(ValueError, r.loop_filtered, -1, boom, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) finally: os.unlink(path) def test_loop_filtered_on_closed(self): path = G.temp_pcap([G.dns_packet("a.com")]) r = pcapy.open_offline(path) r.close() os.unlink(path) self.assertRaises(Exception, r.loop_filtered, -1, lambda h, d, c: None, ADMIT_ALL) def test_bad_callback_type(self): path = G.temp_pcap([G.dns_packet("a.com")]) try: r = pcapy.open_offline(path) self.assertRaises(Exception, r.loop_filtered, -1, "not callable", ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) finally: os.unlink(path) class TestRobustness(unittest.TestCase): """The C path must NEVER crash on malformed/truncated/garbage packets.""" def _nasty_packets(self, seed=0): random.seed(seed) pkts = [b"", b"\x00", b"\xff" * 5, b"\xaa" * 13, b"\xaa" * 14, G.eth(), G.eth() + b"\x45", G.eth() + b"\x45\x00\x00\xff", G.eth() + G.ipv4(17, "1.2.3.4", "5.6.7.8", b"")[:20], # truncated mid-ip G.eth() + struct.pack("B", 0x4f) + b"\x00" * 5, # ihl=15, short G.eth() + struct.pack("B", 0x40) + b"\x00" * 25, # ihl=0 G.eth() + G.ipv4(6, "1.2.3.4", "5.6.7.8", G.tcp(80, 80, 0x18, b""))[:30], G.eth(0x86DD) + b"\x60" + b"\x00" * 5] for _ in range(2000): n = random.randint(0, 80) pkts.append(bytes(bytearray(random.getrandbits(8) for _ in range(n)))) return pkts def test_loop_filtered_survives_garbage(self): pkts = self._nasty_packets(1) res, _ = run_filtered(pkts, admit_mask=ADMIT_ALL, cutoff=2) self.assertEqual(res[ADM] + res[DROP], len(pkts)) # all accounted, no crash def test_loop_to_buffer_survives_garbage(self): pkts = self._nasty_packets(2) path = G.temp_pcap(pkts) try: buf = bytearray(1 << 20) r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) written, dropped, overflow, used = res[:4] self.assertEqual(written + dropped + overflow, len(pkts)) self.assertTrue(used <= len(buf)) finally: os.unlink(path) def test_empty_pcap(self): res, _ = run_filtered([]) self.assertEqual(sum(res), 0) class TestDatalinkOffset(unittest.TestCase): """loop_filtered/loop_to_buffer must classify correctly on non-Ethernet datalinks via the l2_offset parameter (LINUX_SLL='any' iface, DLT_RAW).""" def _ip_dns(self): # bare IPv4/UDP/53 (no L2) return G.ipv4(17, "10.0.0.5", "8.8.8.8", G.udp(12345, 53, G.dns_query("sll.example.com"))) def test_sll_with_offset16(self): pkts = [G.sll() + self._ip_dns()] * 5 path = G.temp_pcap(pkts, linktype=G.DLT_LINUX_SLL) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 16, 1) self.assertEqual(res[DNS], 5) finally: os.unlink(path) def test_sll_wrong_offset_misclassifies(self): # with the default Ethernet offset (14) on an SLL capture, DNS is NOT detected pkts = [G.sll() + self._ip_dns()] * 5 path = G.temp_pcap(pkts, linktype=G.DLT_LINUX_SLL) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) # default l2=14 self.assertEqual(res[DNS], 0) # proves the offset matters (and no crash) finally: os.unlink(path) def test_raw_ip_offset0(self): pkts = [self._ip_dns()] * 4 path = G.temp_pcap(pkts, linktype=G.DLT_RAW) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 0, 1) self.assertEqual(res[DNS], 4) finally: os.unlink(path) def test_sll_ioc_and_buffer(self): ip = G.ipv4(17, "10.0.0.5", G.BAD_IPS[0], G.udp(4444, 5000, b"\x00" * 8)) pkts = [G.sll() + ip] * 3 path = G.temp_pcap(pkts, linktype=G.DLT_LINUX_SLL) try: buf = bytearray(1 << 16) r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, ADMIT_ALL, IOCSET, 0, None, None, 53, 16, 1) # loop_to_buffer tuple: (written,dropped,overflow,used,dns,dpi,noise,ioc,head) self.assertEqual(res[7], 3) # ioc count finally: os.unlink(path) def test_vlan_single_tag_autoskip(self): # default Ethernet l2=14; one 802.1Q tag -> classifier auto-skips to IP at 18 ip = G.ipv4(17, "10.0.0.5", "8.8.8.8", G.udp(1, 53, G.dns_query("vlan.example.com"))) pkts = [G.eth_vlan() + ip] * 5 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) # default l2=14 self.assertEqual(res[DNS], 5) finally: os.unlink(path) def test_vlan_double_tag_qinq(self): ip = G.ipv4(17, "10.0.0.5", "8.8.8.8", G.udp(1, 53, G.dns_query("qinq.example.com"))) pkts = [G.eth_vlan(tags=2) + ip] * 3 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) self.assertEqual(res[DNS], 3) finally: os.unlink(path) def test_vlan_ioc_sip_correct_after_skip(self): # IOC match must use the VLAN-resolved IP offset (sip/dip), not the base offset ip = G.ipv4(17, G.BAD_IPS[0], "10.0.0.5", G.udp(4444, 5000, b"\x00" * 8)) pkts = [G.eth_vlan() + ip] * 2 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 14, 1) self.assertEqual(res[IOC], 2) finally: os.unlink(path) def test_l2_offset_out_of_range(self): path = G.temp_pcap([G.dns_packet("a.com")]) try: r = pcapy.open_offline(path) self.assertRaises(ValueError, r.loop_filtered, -1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 0, None, None, 53, 999) finally: os.unlink(path) class TestSoundSuperset(unittest.TestCase): """Gaps closed so the classifier is a sound superset of Maltrail _process_packet: SYN admitted (heuristics), and IOC matches ANY IPv4 protocol incl ICMP.""" def test_pure_syn_classified(self): res, _ = run_filtered([G.syn_packet()] * 5) self.assertEqual(res[SYN], 5) self.assertEqual(res[NOISE], 0) def test_syn_any_port_counted(self): # SYN to an arbitrary port (not a DPI port) must still be SYN (scan heuristics count all) res, _ = run_filtered([G.syn_packet(dport=33333)]) self.assertEqual(res[SYN], 1) def test_syn_ack_not_syn_class(self): # SYN+ACK (0x12) is not pure SYN -> not class 5 (handled by IP path / IOC instead) pkt = G.eth() + G.ipv4(6, "10.0.0.5", "1.2.3.4", G.tcp(40000, 80, 0x12, b"")) res, _ = run_filtered([pkt]) self.assertEqual(res[SYN], 0) self.assertEqual(res[NOISE], 1) def test_icmp_to_trail_ip_is_ioc(self): res, _ = run_filtered([G.icmp_packet(G.BAD_IPS[0])] * 3) self.assertEqual(res[IOC], 3) # ICMP to bad IP kept (was missed before) self.assertEqual(res[NOISE], 0) def test_icmp_non_trail_is_noise(self): res, _ = run_filtered([G.icmp_packet("9.9.9.9")]) self.assertEqual(res[NOISE], 1) self.assertEqual(res[IOC], 0) def test_icmp_from_trail_ip_is_ioc(self): pkt = G.eth() + G.ipv4(1, G.BAD_IPS[1], "10.0.0.5", b"\x00\x00\x00\x00\x00\x00") res, _ = run_filtered([pkt]) self.assertEqual(res[IOC], 1) def test_inert_packet_still_dropped(self): # plain TCP ACK to a non-trail IP, no PSH/SYN -> truly inert -> noise (safe to drop) pkt = G.eth() + G.ipv4(6, "10.0.0.5", "9.9.9.9", G.tcp(40000, 12345, 0x10, b"\x00" * 20)) res, _ = run_filtered([pkt]) self.assertEqual(res[NOISE], 1) def test_syn_admit_mask(self): # SYN dropped when its bit (5) is not in the mask res, _ = run_filtered([G.syn_packet()], admit_mask=(1 << C_DNS)) self.assertEqual(res[ADM], 0) self.assertEqual(res[SYN], 1) # still classified, just not admitted class TestThreadStress(unittest.TestCase): """Concurrent loop_filtered on independent handles must not crash and must classify correctly (each handle has its own context; the GIL serializes the Python callbacks).""" def test_concurrent_independent_handles(self): counts = [0] * 50 per = [G.dns_packet("a.com")] * 100 + [G.quicish_packet()] * 50 path = G.temp_pcap(per) try: errors = [] def work(idx): try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_ALL, IOCSET, 2, None, None, 53, 14, 1) counts[idx] = res[DNS] except Exception as e: # noqa errors.append(e) ts = [threading.Thread(target=work, args=(i,)) for i in range(8)] for t in ts: t.start() for t in ts: t.join() self.assertEqual(errors, []) self.assertTrue(all(counts[i] == 100 for i in range(8))) finally: os.unlink(path) class TestClassicAPI(unittest.TestCase): def test_next_loop_dispatch(self): pkts = [G.dns_packet("a.com")] * 20 path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) n = 0 while True: hdr, data = r.next() if hdr is None: break n += 1 self.assertEqual(n, 20) r2 = pcapy.open_offline(path) c = [0] r2.loop(-1, lambda h, d: c.__setitem__(0, c[0] + 1)) self.assertEqual(c[0], 20) finally: os.unlink(path) def test_setfilter(self): pkts = [G.dns_packet("a.com"), G.http_packet(), G.quicish_packet()] path = G.temp_pcap(pkts) try: r = pcapy.open_offline(path) r.setfilter("udp port 53") c = [0] r.loop(-1, lambda h, d: c.__setitem__(0, c[0] + 1)) self.assertEqual(c[0], 1) finally: os.unlink(path) def test_datalink(self): path = G.temp_pcap([G.dns_packet("a.com")]) try: self.assertEqual(pcapy.open_offline(path).datalink(), G.DLT_EN10MB) finally: os.unlink(path) # --------------------------------------------------------------------------------------- # # addr_set (generic profile, class 2 = SET_MATCH) # # The set is an open-addressing hash table keyed on the raw network-order address. These # tests pin down its *behaviour* -- exact membership for sequential/CIDR-style ranges, for # random sets, and for the awkward inputs (duplicates, the 0.0.0.0 sentinel, both directions) # -- for set sizes from a handful to a million. They assert results only, never timing, so # they are stable on a loaded machine or a slow runner. # --------------------------------------------------------------------------------------- # C_SET_MATCH = 2 ADMIT_SET_ONLY = 1 << C_SET_MATCH def _ip2int(s): a, b, c, d = (int(x) for x in s.split(".")) return (a << 24) | (b << 16) | (c << 8) | d def _int2ip(v): return "%d.%d.%d.%d" % ((v >> 24) & 0xFF, (v >> 16) & 0xFF, (v >> 8) & 0xFF, v & 0xFF) def _packed(ints): return b"".join(struct.pack("!I", v) for v in ints) def _run_set(packets, addr_set_bytes): """Run the generic profile admitting SET_MATCH only; return (admitted, dropped).""" path = G.temp_pcap(packets) try: r = pcapy.open_offline(path) res = r.loop_filtered(-1, lambda h, d, c: None, ADMIT_SET_ONLY, addr_set_bytes, 0) return res[0], res[1] finally: os.unlink(path) def _pkt(src, dst): return G.eth() + G.ipv4(6, src, dst, G.tcp(1234, 80, 0x10, b"x" * 4)) class TestAddrSetMembership(unittest.TestCase): SEQ_BASE = _ip2int("198.18.0.0") def _sequential(self, size): return [self.SEQ_BASE + i for i in range(size)] def _random(self, size, seed): rnd = random.Random(seed) out, seen = [], set() while len(out) < size: v = rnd.randrange(1, 0xFFFFFFFF) if v not in seen: seen.add(v) out.append(v) return out def _check(self, ips, size): """Members at the edges and the middle of the set must match; neighbours must not.""" members = [ips[0], ips[size // 2], ips[-1]] outside = [self.SEQ_BASE - 1, self.SEQ_BASE + size + 1, _ip2int("10.11.12.13")] outside = [v for v in outside if v not in set(ips)] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in members] packets += [_pkt("10.0.0.9", _int2ip(v)) for v in members] # dst side too packets += [_pkt(_int2ip(v), "10.0.0.9") for v in outside] admitted, dropped = _run_set(packets, _packed(ips)) self.assertEqual(admitted, 2 * len(members)) self.assertEqual(dropped, len(outside)) def test_sequential_10(self): self._check(self._sequential(10), 10) def test_sequential_1000(self): self._check(self._sequential(1000), 1000) def test_sequential_100000(self): # a /16 plus change: the case that used to collapse into one probe chain self._check(self._sequential(100000), 100000) def test_sequential_1000000(self): self._check(self._sequential(1000000), 1000000) def test_random_1000(self): self._check(self._random(1000, seed=11), 1000) def test_random_100000(self): self._check(self._random(100000, seed=12), 100000) def test_full_cidr_block_every_address_matches(self): """Every address of an expanded /20 matches, and the addresses either side do not.""" base = _ip2int("203.0.112.0") block = [base + i for i in range(4096)] probes = [block[0], block[1], block[2048], block[4094], block[-1]] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in probes] packets.append(_pkt(_int2ip(base - 1), "10.0.0.9")) packets.append(_pkt(_int2ip(base + 4096), "10.0.0.9")) admitted, dropped = _run_set(packets, _packed(block)) self.assertEqual(admitted, len(probes)) self.assertEqual(dropped, 2) def test_many_disjoint_cidr_blocks(self): """Sixteen /24s scattered across the address space, all members must be found.""" blocks = [] for i in range(16): base = _ip2int("%d.%d.0.0" % (11 + i * 7, i * 3 + 1)) blocks.extend(base + j for j in range(256)) probes = blocks[::137] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in probes] admitted, dropped = _run_set(packets, _packed(blocks)) self.assertEqual(admitted, len(probes)) self.assertEqual(dropped, 0) def test_duplicates_and_zero_sentinel_ignored(self): """0.0.0.0 is the empty-slot sentinel and duplicates are harmless.""" ips = self._sequential(1000) noisy = ips + ips[:500] + [0, 0, 0] probes = [ips[0], ips[499], ips[-1]] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in probes] packets.append(_pkt("10.0.0.9", "10.0.0.8")) admitted, dropped = _run_set(packets, _packed(noisy)) self.assertEqual(admitted, len(probes)) self.assertEqual(dropped, 1) def test_sequential_and_random_mixed(self): ips = self._sequential(50000) + self._random(50000, seed=13) probes = [ips[0], ips[49999], ips[50000], ips[-1]] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in probes] packets.append(_pkt("172.31.255.254", "10.0.0.9")) admitted, dropped = _run_set(packets, _packed(ips)) self.assertEqual(admitted, len(probes)) self.assertEqual(dropped, 1) def test_empty_set_admits_nothing(self): packets = [_pkt("198.18.0.1", "10.0.0.9")] * 3 admitted, dropped = _run_set(packets, b"") self.assertEqual(admitted, 0) self.assertEqual(dropped, 3) def test_ipv6_never_matches_set(self): ips = self._sequential(1000) packets = [G.eth(0x86DD) + G.ipv6(17, "2001:db8::1", "2001:db8::2", G.udp(1234, 53, b"z" * 8))] admitted, dropped = _run_set(packets, _packed(ips)) self.assertEqual(admitted, 0) self.assertEqual(dropped, 1) def test_set_match_through_vlan_tag(self): ips = self._sequential(1000) inner = G.ipv4(6, _int2ip(ips[500]), "10.0.0.9", G.tcp(1234, 80, 0x10, b"y" * 4)) packets = [G.eth_vlan() + inner] admitted, _ = _run_set(packets, _packed(ips)) self.assertEqual(admitted, 1) def test_to_buffer_agrees_with_loop_filtered(self): ips = self._sequential(100000) probes = [ips[0], ips[50000], ips[-1]] packets = [_pkt(_int2ip(v), "10.0.0.9") for v in probes] packets.append(_pkt("10.0.0.1", "10.0.0.9")) path = G.temp_pcap(packets) try: buf = bytearray(1 << 16) r = pcapy.open_offline(path) res = r.loop_to_buffer(-1, buf, ADMIT_SET_ONLY, _packed(ips), 0) self.assertEqual(res[0], len(probes)) # written self.assertEqual(res[1], 1) # dropped self.assertEqual(res[2], 0) # overflow finally: os.unlink(path) class TestKeywordArguments(unittest.TestCase): """The docstrings and the README document these calls with keyword arguments, so they have to actually accept them (both methods used to be METH_VARARGS: flow_cutoff=3 -> TypeError).""" def setUp(self): pkts = [G.dns_packet("example.com"), G.http_packet(), G.syn_packet()] * 4 self.path = G.temp_pcap(pkts) def tearDown(self): os.unlink(self.path) def test_loop_filtered_all_keywords(self): kw = dict(admit_mask=ADMIT_ALL, addr_set=IOCSET, flow_cutoff=3, dpi_ports=None, tls_ports=None, dns_port=53, l2_offset=14, profile=1) cb = lambda hdr, data, cls: None by_kw = pcapy.open_offline(self.path).loop_filtered(-1, cb, **kw) by_pos = pcapy.open_offline(self.path).loop_filtered( -1, cb, ADMIT_ALL, IOCSET, 3, None, None, 53, 14, 1) self.assertEqual(by_kw, by_pos) def test_loop_filtered_cnt_and_cb_by_keyword(self): seen = [] res = pcapy.open_offline(self.path).loop_filtered( cnt=-1, cb=lambda hdr, data, cls: seen.append(cls), flow_cutoff=3) self.assertEqual(res[0], len(seen)) def test_loop_to_buffer_all_keywords(self): kw = dict(admit_mask=ADMIT_ALL, addr_set=IOCSET, flow_cutoff=3, dpi_ports=None, tls_ports=None, dns_port=53, l2_offset=14, profile=1) b1, b2 = bytearray(1 << 16), bytearray(1 << 16) by_kw = pcapy.open_offline(self.path).loop_to_buffer(-1, b1, **kw) by_pos = pcapy.open_offline(self.path).loop_to_buffer( -1, b2, ADMIT_ALL, IOCSET, 3, None, None, 53, 14, 1) self.assertEqual(by_kw, by_pos) self.assertEqual(b1, b2) def test_unknown_keyword_rejected(self): r = pcapy.open_offline(self.path) self.assertRaises(TypeError, r.loop_filtered, -1, lambda h, d, c: None, no_such_arg=1) def test_duplicate_argument_rejected(self): r = pcapy.open_offline(self.path) self.assertRaises(TypeError, r.loop_filtered, -1, lambda h, d, c: None, 7, admit_mask=7) class TestFlowKey(unittest.TestCase): """The flow table folds a >96-bit 5-tuple into a 64-bit key. A SLOT collision only restarts a counter (an extra head, documented), but a KEY collision would make two flows share one counter and hide a real flow head -- the one thing the design must not do.""" @staticmethod def _flows(n): packets = [] for i in range(n): src = "10.%d.%d.%d" % ((i >> 16) & 0xff, (i >> 8) & 0xff, i & 0xff) dst = "203.0.113.%d" % (i & 0xff) sport, dport = 1024 + (i % 64000), 443 for _ in range(3): # 3 packets on every flow packets.append(G.eth() + G.ipv4(6, src, dst, G.tcp(sport, dport, 0x10, b"x" * 8))) return packets def _heads(self, packets, cutoff): path = G.temp_pcap(packets) try: r = pcapy.open_offline(path) # generic profile: bit 1 = FLOW_HEAD res = r.loop_filtered(-1, lambda h, d, c: None, admit_mask=(1 << 1), flow_cutoff=cutoff, profile=0) return res[0] finally: os.unlink(path) def test_no_flow_head_is_suppressed(self): n = 4000 packets = self._flows(n) self.assertGreaterEqual(self._heads(packets, 1), n) # never fewer than one per flow self.assertGreaterEqual(self._heads(packets, 3), 3 * n) # all 3 packets are heads def test_distinct_ports_are_distinct_flows(self): # same addresses, ports differing only in the high bit of the source port: these must not # collapse onto one key (a plain `sport << 16` fold also invoked signed-shift UB here). packets = [] for sport in (0x7fff, 0x8000, 0xffff, 0x0001): packets.append(G.eth() + G.ipv4(6, "10.0.0.1", "10.0.0.2", G.tcp(sport, 443, 0x10, b"y" * 8))) self.assertEqual(self._heads(packets, 1), 4) def test_reversed_direction_is_a_separate_flow(self): a = G.eth() + G.ipv4(6, "10.0.0.1", "10.0.0.2", G.tcp(1111, 2222, 0x10, b"z" * 8)) b = G.eth() + G.ipv4(6, "10.0.0.2", "10.0.0.1", G.tcp(2222, 1111, 0x10, b"z" * 8)) self.assertEqual(self._heads([a, b], 1), 2) if __name__ == "__main__": unittest.main() pcapy-ng-2.1.0/tests/wheel_smoketest.py000066400000000000000000000026541524067405300201630ustar00rootroot00000000000000#!/usr/bin/env python3 """Smoke test run against each cibuildwheel-built wheel in an isolated environment. Confirms the installed wheel (a) imports, (b) reads a pcap via the classic API, and (c) exposes the fast-path additions (loop_filtered / loop_to_buffer / filtered_stats). Usage: python wheel_smoketest.py """ import sys import pcapy def main(pcap_path): # classic API still works r = pcapy.open_offline(pcap_path) count = [0] r.loop(-1, lambda hdr, data: count.__setitem__(0, count[0] + 1)) assert count[0] > 0, "loop() read zero packets from %s" % pcap_path print("loop() read %d packets" % count[0]) # fast-path additions present r2 = pcapy.open_offline(pcap_path) for name in ("loop_filtered", "loop_to_buffer", "filtered_stats", "next_batch"): assert hasattr(r2, name), "missing fast-path method: %s" % name print("fast-path API present: loop_filtered, loop_to_buffer, filtered_stats, next_batch") # exercise loop_filtered end to end (admit everything so any pcap yields output) r3 = pcapy.open_offline(pcap_path) seen = [0] res = r3.loop_filtered(-1, lambda hdr, data, cls: seen.__setitem__(0, seen[0] + 1), 0xFF) assert sum(res) > 0, "loop_filtered classified zero packets" print("loop_filtered OK: %r" % (res,)) print("WHEEL SMOKE TEST PASSED") if __name__ == "__main__": main(sys.argv[1] if len(sys.argv) > 1 else "tests/96pings.pcap") pcapy-ng-2.1.0/win32/000077500000000000000000000000001524067405300142005ustar00rootroot00000000000000pcapy-ng-2.1.0/win32/dllmain.cc000077500000000000000000000011651524067405300161350ustar00rootroot00000000000000/* * Copyright (c) 2021-2026 Miroslav Stampar (contact@sekuripy.hr) * Copyright (c) 2014 CORE Security Technologies * * This software is provided under under the Apache Software License. * See the accompanying LICENSE file for more information. * * Win32 DLL entry point. * */ #include BOOL APIENTRY DllMain( HANDLE hModule, DWORD ul_reason_for_call, LPVOID lpReserved ) { switch (ul_reason_for_call) { case DLL_PROCESS_ATTACH: case DLL_THREAD_ATTACH: case DLL_THREAD_DETACH: case DLL_PROCESS_DETACH: break; } return TRUE; }