codspeed-4.4.1/.cargo_vcs_info.json0000644000000001550000000000100126310ustar { "git": { "sha1": "dccf5f4a5d4ad018905018be4a9d7f56141b4268" }, "path_in_vcs": "crates/codspeed" }codspeed-4.4.1/Cargo.lock0000644000000305500000000000100106060ustar # This file is automatically @generated by Cargo. # It is not intended for manual editing. version = 3 [[package]] name = "anyhow" version = "1.0.97" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcfed56ad506cb2c684a14971b8861fdc3baaaae314b9e5f9bb532cbe3ba7a4f" [[package]] name = "approx" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cab112f0a86d568ea0e627cc1d6be74a1e9cd55214684db5561995f6dad897c6" dependencies = [ "num-traits", ] [[package]] name = "autocfg" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" [[package]] name = "cc" version = "1.2.38" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "80f41ae168f955c12fb8960b057d70d0ca153fb83182b57d86380443527be7e9" dependencies = [ "find-msvc-tools", "shlex", ] [[package]] name = "cfg-if" version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "cfg_aliases" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "codspeed" version = "4.4.1" dependencies = [ "anyhow", "cc", "colored", "getrandom", "glob", "libc", "nix", "serde", "serde_json", "statrs", "tempfile", ] [[package]] name = "colored" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cbf2150cce219b664a8a70df7a1f933836724b503f8a413af9365b4dcc4d90b8" dependencies = [ "lazy_static", "windows-sys 0.48.0", ] [[package]] name = "errno" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" dependencies = [ "libc", "windows-sys 0.52.0", ] [[package]] name = "fastrand" version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" [[package]] name = "find-msvc-tools" version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" [[package]] name = "getrandom" version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" dependencies = [ "cfg-if", "libc", "wasi", ] [[package]] name = "glob" version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" [[package]] name = "itoa" version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" [[package]] name = "lazy_static" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" [[package]] name = "linux-raw-sys" version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" [[package]] name = "nix" version = "0.31.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5d6d0705320c1e6ba1d912b5e37cf18071b6c2e9b7fa8215a1e8a7651966f5d3" dependencies = [ "bitflags 2.6.0", "cfg-if", "cfg_aliases", "libc", ] [[package]] name = "num-traits" version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", ] [[package]] name = "proc-macro2" version = "1.0.93" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60946a68e5f9d28b0dc1c21bb8a97ee7d018a8b322fa57838ba31cc878e22d99" dependencies = [ "unicode-ident", ] [[package]] name = "quote" version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7" dependencies = [ "proc-macro2", ] [[package]] name = "redox_syscall" version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "567664f262709473930a4bf9e51bf2ebf3348f2e748ccc50dea20646858f8f29" dependencies = [ "bitflags 1.3.2", ] [[package]] name = "rustix" version = "0.38.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" dependencies = [ "bitflags 2.6.0", "errno", "libc", "linux-raw-sys", "windows-sys 0.52.0", ] [[package]] name = "ryu" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "serde" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ "serde_core", "serde_derive", ] [[package]] name = "serde_core" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", "syn", ] [[package]] name = "serde_json" version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", "serde_core", ] [[package]] name = "shlex" version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "statrs" version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a3fe7c28c6512e766b0874335db33c94ad7b8f9054228ae1c2abd47ce7d335e" dependencies = [ "approx", "num-traits", ] [[package]] name = "syn" version = "2.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5d0adab1ae378d7f53bdebc67a39f1f151407ef230f0ce2883572f5d8985c80" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] [[package]] name = "tempfile" version = "3.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" dependencies = [ "cfg-if", "fastrand", "redox_syscall", "rustix", "windows-sys 0.48.0", ] [[package]] name = "unicode-ident" version = "1.0.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" [[package]] name = "windows-sys" version = "0.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" dependencies = [ "windows-targets 0.48.5", ] [[package]] name = "windows-sys" version = "0.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" dependencies = [ "windows-targets 0.52.6", ] [[package]] name = "windows-targets" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" dependencies = [ "windows_aarch64_gnullvm 0.48.5", "windows_aarch64_msvc 0.48.5", "windows_i686_gnu 0.48.5", "windows_i686_msvc 0.48.5", "windows_x86_64_gnu 0.48.5", "windows_x86_64_gnullvm 0.48.5", "windows_x86_64_msvc 0.48.5", ] [[package]] name = "windows-targets" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" [[package]] name = "windows_aarch64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" [[package]] name = "windows_aarch64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_i686_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" [[package]] name = "windows_i686_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" [[package]] name = "windows_i686_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_x86_64_gnu" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" [[package]] name = "windows_x86_64_gnu" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" [[package]] name = "windows_x86_64_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_msvc" version = "0.48.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "windows_x86_64_msvc" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" codspeed-4.4.1/Cargo.toml0000644000000033670000000000100106370ustar # THIS FILE IS AUTOMATICALLY GENERATED BY CARGO # # When uploading crates to the registry Cargo will automatically # "normalize" Cargo.toml files for maximal compatibility # with all versions of Cargo and also rewrite `path` dependencies # to registry (e.g., crates.io) dependencies. # # If you are reading this file be aware that the original Cargo.toml # will likely look very different (and much more reasonable). # See Cargo.toml.orig for the original contents. [package] edition = "2021" rust-version = "1.74" name = "codspeed" version = "4.4.1" authors = ["Arthur Pastel "] build = "build.rs" autolib = false autobins = false autoexamples = false autotests = false autobenches = false description = "Core instrumentation library for CodSpeed" homepage = "https://codspeed.io" documentation = "https://codspeed.io/docs/reference/codspeed-rust" readme = "README.md" keywords = [ "codspeed", "benchmark", ] categories = [ "development-tools", "development-tools::profiling", "development-tools::testing", ] license = "MIT OR Apache-2.0" repository = "https://github.com/CodSpeedHQ/codspeed-rust" [lib] name = "codspeed" path = "src/lib.rs" [[bench]] name = "native" path = "benches/native.rs" harness = false [dependencies.anyhow] version = "1.0.97" [dependencies.colored] version = "2.0.0" [dependencies.getrandom] version = "0.2" [dependencies.glob] version = "0.3.2" [dependencies.libc] version = "^0.2" [dependencies.nix] version = "0.31.1" features = ["time"] [dependencies.serde] version = "1.0.217" features = ["derive"] [dependencies.serde_json] version = "1.0.138" [dependencies.statrs] version = "0.18.0" default-features = false [dev-dependencies.tempfile] version = "3.7.0" [build-dependencies.cc] version = "1.0" codspeed-4.4.1/Cargo.toml.orig000064400000000000000000000017101046102023000143060ustar 00000000000000[package] name = "codspeed" version = "4.4.1" rust-version = "1.74" # MSRV edition = "2021" description = "Core instrumentation library for CodSpeed" authors = ["Arthur Pastel "] documentation = "https://codspeed.io/docs/reference/codspeed-rust" readme = "README.md" repository = "https://github.com/CodSpeedHQ/codspeed-rust" homepage = "https://codspeed.io" license = "MIT OR Apache-2.0" categories = [ "development-tools", "development-tools::profiling", "development-tools::testing", ] keywords = ["codspeed", "benchmark"] [dependencies] anyhow = { workspace = true } colored = "2.0.0" glob = "0.3.2" libc = "^0.2" nix = { version = "0.31.1", features = ["time"] } getrandom = "0.2" serde = { workspace = true } serde_json = { workspace = true } statrs = { version = "0.18.0", default-features = false } [[bench]] name = "native" harness = false [dev-dependencies] tempfile = { workspace = true } [build-dependencies] cc = "1.0" codspeed-4.4.1/README.md000064400000000000000000000013761046102023000127060ustar 00000000000000

codspeed

[![CI](https://github.com/CodSpeedHQ/codspeed-rust/actions/workflows/ci.yml/badge.svg?branch=main)](https://github.com/CodSpeedHQ/codspeed-rust/actions/workflows/ci.yml) [![Crates.io](https://img.shields.io/crates/v/codspeed)](https://crates.io/crates/codspeed) [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) [![CodSpeed Badge](https://img.shields.io/endpoint?url=https://codspeed.io/badge.json)](https://codspeed.io/CodSpeedHQ/codspeed-rust) The core library used to integrate with Codspeed runners
For now, this crate should not be used directly. Instead, use one of the integration crates: - `codspeed-bencher-compat` - `codspeed-criterion-compat` codspeed-4.4.1/benches/native.rs000064400000000000000000000025261046102023000146700ustar 00000000000000use std::collections::HashMap; use codspeed::{codspeed::black_box, codspeed_bench, codspeed_main}; pub fn fibonacci_recursive_cached(n: u64) -> u64 { fn inner(n: u64, cache: &mut HashMap) -> u64 { match n { 0 | 1 => 1, _ => { if cache.contains_key(&n) { *cache.get(&n).unwrap() } else { let result = inner(n - 1, cache) + inner(n - 2, cache); cache.insert(n, result); result } } } } let mut cache = HashMap::new(); inner(n, &mut cache) } pub fn fibonacci_recursive(n: i32) -> u64 { match n { 0 | 1 => 1, _ => fibonacci_recursive(n - 1) + fibonacci_recursive(n - 2), } } pub fn fibonacci_iterative(n: u64) -> u64 { if n <= 1 { return n; } let mut sum = 0; let mut last = 0; let mut curr = 1; for _i in 1..n { sum = last + curr; last = curr; curr = sum; } sum } codspeed_bench!(fibo_recursive, || fibonacci_recursive(black_box(10))); codspeed_bench!(fibo_iterative, || fibonacci_iterative(black_box(10))); codspeed_bench!(fibo_recursive_cached, || fibonacci_recursive_cached( black_box(10) )); codspeed_main!(fibo_recursive, fibo_iterative, fibo_recursive_cached); codspeed-4.4.1/build.rs000064400000000000000000000035361046102023000130740ustar 00000000000000fn main() { println!("cargo:rustc-check-cfg=cfg(use_instrument_hooks)"); println!("cargo:rerun-if-changed=instrument-hooks/dist/core.c"); println!("cargo:rerun-if-changed=instrument-hooks/includes/core.h"); println!("cargo:rerun-if-changed=build.rs"); let mut build = cc::Build::new(); build .flag("-std=c11") .file("instrument-hooks/dist/core.c") .include("instrument-hooks/includes") // We generated the C code from Zig, which contains some warnings // that can be safely ignored. .flag("-Wno-format") .flag("-Wno-format-security") .flag("-Wno-unused-but-set-variable") .flag("-Wno-unused-const-variable") .flag("-Wno-type-limits") .flag("-Wno-uninitialized") // Ignore warnings when cross-compiling: .flag("-Wno-overflow") .flag("-Wno-unused-function") .flag("-Wno-constant-conversion") .flag("-Wno-incompatible-pointer-types") // Disable warnings, as we will have lots of them .warnings(false) .extra_warnings(false) .cargo_warnings(false) .opt_level(3); let result = build.try_compile("instrument_hooks"); match result { Ok(_) => println!("cargo:rustc-cfg=use_instrument_hooks"), Err(e) => { let compiler = build.try_get_compiler().expect("Failed to get C compiler"); eprintln!("\n\nWARNING: Failed to compile instrument-hooks native library with cc-rs."); eprintln!("The library will still compile, but instrument-hooks functionality will be disabled."); eprintln!("Compiler information: {compiler:?}"); eprintln!("Compilation error: {e}\n"); println!("cargo:warning=Failed to compile instrument-hooks native library with cc-rs. Continuing with noop implementation."); } } } codspeed-4.4.1/instrument-hooks/.clang-format000064400000000000000000000001121046102023000173160ustar 00000000000000--- Language: Cpp BasedOnStyle: Google PointerAlignment: Left ... codspeed-4.4.1/instrument-hooks/.github/workflows/ci.yml000064400000000000000000000173231046102023000214720ustar 00000000000000name: CI on: push: branches: [main] pull_request: workflow_dispatch: env: COMMON_CFLAGS: >- -Wall -Werror -Wno-format -Wno-format-security -Wno-unused-but-set-variable -Wno-unused-const-variable -Wno-type-limits -Wno-uninitialized jobs: lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - uses: mlugg/setup-zig@v2 - uses: pre-commit/action@v3.0.1 with: extra_args: --all-files tests: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: mlugg/setup-zig@v2 - uses: extractions/setup-just@v3 - name: Run tests run: zig build test --summary all # Skip installing package docs to avoid wasting time when installing valgrind # See: https://github.com/actions/runner-images/issues/10977#issuecomment-2810713336 - name: Skip installing package docs if: runner.os == 'Linux' run: | sudo tee /etc/dpkg/dpkg.cfg.d/01_nodoc > /dev/null << 'EOF' path-exclude /usr/share/doc/* path-exclude /usr/share/man/* path-exclude /usr/share/info/* EOF - name: Run valgrind tests if: runner.os == 'Linux' run: | sudo apt-get update sudo apt-get install -y valgrind just test-valgrind build: strategy: matrix: os: [ubuntu-latest, ubuntu-24.04-arm, macos-latest] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: mlugg/setup-zig@v2 - name: Build run: zig build test-build-cmake: strategy: matrix: include: - name: Linux GCC os: ubuntu-latest compiler: gcc - name: Linux Clang os: ubuntu-latest compiler: clang - name: macOS Clang os: macos-latest compiler: clang - name: Windows MSVC os: windows-latest compiler: cl runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - uses: mlugg/setup-zig@v2 - uses: extractions/setup-just@v3 - name: Cache build uses: actions/cache@v4 with: path: example/build key: ${{ runner.os }}-build-${{ hashFiles('**/CMakeLists.txt') }} - name: "Enable MSVC command prompt" if: matrix.os == 'windows-latest' uses: ilammy/msvc-dev-cmd@v1 - name: "Install cmake" uses: lukka/get-cmake@latest - name: "Build debug mode" shell: bash run: | cd example mkdir -p out cd out cmake .. -DCMAKE_C_COMPILER=${{ matrix.compiler }} -DCMAKE_BUILD_TYPE=Debug cmake --build . --target example - name: "Run example (Windows)" if: runner.os == 'Windows' shell: bash run: | cd example/out/Debug ./example.exe - name: "Run example (Unix)" if: runner.os != 'Windows' shell: bash run: | cd example/out ./example test-build-bazel: strategy: matrix: include: - name: Linux os: ubuntu-latest - name: macOS os: macos-latest - name: Windows os: windows-latest runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v4 - name: Set up Bazel uses: bazel-contrib/setup-bazel@0.14.0 with: # Avoid downloading Bazel every time. bazelisk-cache: true # Store build cache per workflow. disk-cache: ${{ github.workflow }} # Share repository cache between workflows. repository-cache: true # Symlinks on windows are broken with bazel - name: Fix symlink on Windows if: runner.os == 'Windows' shell: powershell run: | Get-ChildItem -Path example/instrument-hooks -Force | Remove-Item -Recurse -Force -ErrorAction SilentlyContinue Copy-Item -Path dist -Destination example/instrument-hooks/dist -Recurse Copy-Item -Path includes -Destination example/instrument-hooks/includes -Recurse - name: Build benchmarks working-directory: example run: | bazel build //:example - name: Run benchmarks working-directory: example run: | bazel run //:example test-gcc-versions: runs-on: ubuntu-latest strategy: fail-fast: false matrix: version: [15, 14, 13, 12, 11, 10, 9] container: image: gcc:${{ matrix.version }} steps: - uses: actions/checkout@v4 - name: Build example with GCC ${{ matrix.version }} run: | gcc -std=c11 -O3 example/main.c dist/core.c \ -I includes/ \ ${{ env.COMMON_CFLAGS }} \ -o test-example - name: Run example run: ./test-example test-clang-versions: runs-on: ubuntu-latest strategy: fail-fast: false matrix: version: [19, 18, 17, 16, 15, 14, 13] container: image: silkeh/clang:${{ matrix.version }} steps: - uses: actions/checkout@v4 - name: Build example with Clang ${{ matrix.version }} run: | clang -std=c11 -O3 example/main.c dist/core.c \ -I includes/ \ ${{ env.COMMON_CFLAGS }} \ -o test-example - name: Run example run: ./test-example cross-compile: runs-on: ubuntu-latest strategy: fail-fast: false matrix: target: # Linux targets - x86/x64 - { arch: x86_64-linux-gnu, name: "Linux x86_64 (glibc)" } - { arch: x86_64-linux-musl, name: "Linux x86_64 (musl)" } - { arch: x86-linux-gnu, name: "Linux x86 (32-bit)" } # Linux targets - ARM - { arch: aarch64-linux-gnu, name: "Linux ARM64 (glibc)" } - { arch: aarch64-linux-musl, name: "Linux ARM64 (musl)" } - { arch: arm-linux-gnueabihf, name: "Linux ARM (hard-float)" } - { arch: arm-linux-gnueabi, name: "Linux ARM (soft-float)" } # Linux targets - RISC-V - { arch: riscv64-linux-gnu, name: "Linux RISC-V 64 (glibc)" } - { arch: riscv64-linux-musl, name: "Linux RISC-V 64 (musl)" } # Linux targets - Other architectures - { arch: powerpc64le-linux-gnu, name: "Linux PowerPC64 LE" } - { arch: s390x-linux-gnu, name: "Linux s390x (IBM Z)" } - { arch: mips64el-linux-gnuabi64, name: "Linux MIPS64 LE" } - { arch: loongarch64-linux-gnu, name: "Linux LoongArch64" } # macOS targets - { arch: x86_64-macos, name: "macOS x86_64 (Intel)" } - { arch: aarch64-macos, name: "macOS ARM64 (Apple Silicon)" } # Windows targets - { arch: x86_64-windows-gnu, name: "Windows x86_64" } - { arch: x86-windows-gnu, name: "Windows x86 (32-bit)" } steps: - uses: actions/checkout@v4 - uses: mlugg/setup-zig@v2 - name: Compile the example run: | zig cc example/main.c dist/core.c \ -I includes/ \ -target ${{ matrix.target.arch }} \ ${{ env.COMMON_CFLAGS }} \ -Wno-constant-conversion \ -Wno-incompatible-pointer-types \ -Wno-unused-function \ -o example.out check: runs-on: ubuntu-latest if: always() needs: - lint - tests - build - test-build-cmake - test-build-bazel - test-gcc-versions - test-clang-versions - cross-compile steps: - uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJson( needs ) }} codspeed-4.4.1/instrument-hooks/.gitignore000064400000000000000000000001071046102023000167370ustar 00000000000000.zig-cache zig-out target/ ./main **/build # Bazel output **/bazel-* codspeed-4.4.1/instrument-hooks/.pre-commit-config.yaml000064400000000000000000000010251046102023000212300ustar 00000000000000exclude: ^(includes/(valgrind|callgrind|zig)\.h|dist/core\.c) repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.4.0 hooks: - id: trailing-whitespace - id: end-of-file-fixer - id: check-added-large-files - repo: https://github.com/batmac/pre-commit-zig rev: v0.3.0 hooks: - id: zig-fmt - id: zig-build - id: zig-build-test - repo: https://github.com/cpp-linter/cpp-linter-hooks rev: v0.6.1 hooks: - id: clang-format files: \.(c|h)$ codspeed-4.4.1/instrument-hooks/Justfile000064400000000000000000000017231046102023000164640ustar 00000000000000clean: rm -rf zig-out/ rm -rf .zig-cache rm -rf dist/core.c rm -rf example/build build: # Use a fixed traversal seed for reproducible transpilation output zig build --seed 12345 build-example: cd example && mkdir -p build && cd build && cmake .. && make -j cd example && bazelisk build //:example release: build test -f ./zig-out/lib/zig.h || cp "$(zig env | jq -r .lib_dir)/zig.h" ./includes/zig.h mkdir -p dist/ python3 scripts/release.py test: zig build test --summary all test-valgrind: #!/usr/bin/env bash rm -f /tmp/runner*.fifo export CFLAGS="-Wno-error=format -Wno-error=format-security -Wno-format -Wno-format-security" clang $CFLAGS -O3 example/main.c dist/core.c -I includes/ -o clang-main && ./clang-main valgrind ./clang-main gcc $CFLAGS -O3 example/main.c dist/core.c -I includes/ -o gcc-main && ./gcc-main valgrind ./gcc-main rm -rf clang-main gcc-main fmt: zig fmt src codspeed-4.4.1/instrument-hooks/LICENSE-APACHE000064400000000000000000000252771046102023000167120ustar 00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright 2025 CodSpeed Technology SAS Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. codspeed-4.4.1/instrument-hooks/LICENSE-MIT000064400000000000000000000021021046102023000164000ustar 00000000000000The MIT License (MIT) Copyright (c) 2025 CodSpeed Technology SAS Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. codspeed-4.4.1/instrument-hooks/README.md000064400000000000000000000025521046102023000162340ustar 00000000000000

instrument-hooks

[![CI](https://github.com/CodSpeedHQ/instrument-hooks/actions/workflows/ci.yml/badge.svg)](https://github.com/CodSpeedHQ/instrument-hooks/actions/workflows/ci.yml) [![Discord](https://img.shields.io/badge/chat%20on-discord-7289da.svg)](https://discord.com/invite/MxpaCfKSqF) Zig library to control instrumentations via IPC.
## Requirements - **Zig**: 0.14 - [**Just**](https://github.com/casey/just) (optional): To easily run the build, formatter or tests ## How to add new integration? This library is intended to be used as a C library. The main source file is in `dist/core.c` and the headers are in `includes/`. See `examples/main.c` for an example on how to use it. To test if it worked, call `is_instrumented` which should return `false` when running without Codspeed. To run with Codspeed, execute the following: ``` codspeed run -- ``` To make sure your integration is fully working, you have to implement all these hooks: - start_benchmark: Call this when the benchmark starts, to start measuring the performance. - stop_benchmark: Stop measuring the performance after the benchmark stopped. - set_executed_benchmark: Provide metadata about which benchmark was executed. - set_integration: Provide metadata about the integration. ## Run tests ``` zig build test --summary all ``` or ``` just test ``` codspeed-4.4.1/instrument-hooks/build.zig000064400000000000000000000016771046102023000165760ustar 00000000000000const std = @import("std"); pub fn build(b: *std.Build) void { // Core Library // const libcore = b.addStaticLibrary(.{ .name = "core", .root_source_file = b.path("src/c.zig"), .target = b.resolveTargetQuery(.{ .ofmt = .c }), .optimize = .ReleaseSmall, .link_libc = true, .strip = true, .pic = true, }); libcore.no_builtin = true; b.installArtifact(libcore); // Tests // const test_main = b.addTest(.{ .root_source_file = b.path("src/root.zig"), .optimize = .ReleaseSafe, .link_libc = true, .test_runner = .{ .path = b.path("src/tests/runner.zig"), .mode = .simple } }); test_main.addCSourceFile(.{ .file = b.path("src/helpers/valgrind_wrapper.c") }); test_main.addIncludePath(b.path("includes")); test_main.linkLibC(); const run_test_main = b.addRunArtifact(test_main); b.step("test", "test utility functions").dependOn(&run_test_main.step); } codspeed-4.4.1/instrument-hooks/build.zig.zon000064400000000000000000000003651046102023000173740ustar 00000000000000.{ .name = .instrument_hooks, .fingerprint = 0xd871865a9799ec8e, .version = "0.0.1", .minimum_zig_version = "0.14.0", .dependencies = .{}, .paths = .{ "build.zig", "build.zig.zon", "src", }, } codspeed-4.4.1/instrument-hooks/example/.bazelversion000064400000000000000000000000061046102023000211040ustar 000000000000007.6.1 codspeed-4.4.1/instrument-hooks/example/BUILD000064400000000000000000000026431046102023000171730ustar 00000000000000load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library") config_setting( name = "windows", constraint_values = ["@platforms//os:windows"], ) # Instrument-hooks library with warning suppressions cc_library( name = "instrument_hooks", srcs = ["instrument-hooks/dist/core.c"], hdrs = glob(["instrument-hooks/includes/*.h"]), includes = ["instrument-hooks/includes"], copts = select({ ":windows": [ "/wd4101", # unreferenced local variable (equivalent to -Wno-unused-variable) "/wd4189", # local variable is initialized but not referenced (equivalent to -Wno-unused-but-set-variable) "/wd4100", # unreferenced formal parameter (equivalent to -Wno-unused-parameter) "/wd4245", # signed/unsigned mismatch "/wd4132", # const object should be initialized "/wd4146", # unary minus operator applied to unsigned type ], "//conditions:default": [ "-Wno-maybe-uninitialized", "-Wno-unused-variable", "-Wno-unused-parameter", "-Wno-unused-but-set-variable", "-Wno-type-limits", ], }), visibility = ["//visibility:public"], ) cc_binary( name = "example", srcs = ["main.c"], deps = [":instrument_hooks"], copts = select({ ":windows": ["/W4", "/WX"], "//conditions:default": ["-Wall", "-Wextra", "-Werror"], }), ) codspeed-4.4.1/instrument-hooks/example/CMakeLists.txt000064400000000000000000000032661046102023000211530ustar 00000000000000cmake_minimum_required(VERSION 3.16) project(instrument-hooks-example C) set(CMAKE_C_STANDARD 11) set(CMAKE_C_STANDARD_REQUIRED ON) # Define the instrument_hooks library add_library( instrument_hooks STATIC ${CMAKE_CURRENT_SOURCE_DIR}/instrument-hooks/dist/core.c ) target_include_directories( instrument_hooks PUBLIC $ $ ) # Suppress warnings for the instrument_hooks library if(CMAKE_CXX_COMPILER_ID MATCHES "GNU|Clang") target_compile_options( instrument_hooks PRIVATE -Wno-maybe-uninitialized -Wno-unused-variable -Wno-unused-parameter -Wno-unused-but-set-variable -Wno-type-limits ) elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") target_compile_options( instrument_hooks PRIVATE /wd4101 # unreferenced local variable (equivalent to -Wno-unused-variable) /wd4189 # local variable is initialized but not referenced (equivalent to -Wno-unused-but-set-variable) /wd4100 # unreferenced formal parameter (equivalent to -Wno-unused-parameter) /wd4245 # signed/unsigned mismatch /wd4132 # const object should be initialized /wd4146 # unary minus operator applied to unsigned type ) endif() # Example executable add_executable(example main.c) target_link_libraries(example instrument_hooks) # Treat warnings as errors for the example if(CMAKE_C_COMPILER_ID MATCHES "GNU|Clang") target_compile_options(example PRIVATE -Wall -Wextra -Werror) elseif(CMAKE_C_COMPILER_ID STREQUAL "MSVC") target_compile_options(example PRIVATE /W4 /WX) endif() codspeed-4.4.1/instrument-hooks/example/MODULE.bazel000064400000000000000000000002271046102023000204110ustar 00000000000000module(name = "instrument_hooks", version = "1.0.0") bazel_dep(name = "rules_cc", version = "0.0.9") bazel_dep(name = "platforms", version = "0.0.8") codspeed-4.4.1/instrument-hooks/example/MODULE.bazel.lock000064400000000000000000000227021046102023000213420ustar 00000000000000{ "lockFileVersion": 13, "registryFileHashes": { "https://bcr.bazel.build/bazel_registry.json": "8a28e4aff06ee60aed2a8c281907fb8bcbf3b753c91fb5a5c57da3215d5b3497", "https://bcr.bazel.build/modules/abseil-cpp/20210324.2/MODULE.bazel": "7cd0312e064fde87c8d1cd79ba06c876bd23630c83466e9500321be55c96ace2", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/MODULE.bazel": "70390338f7a5106231d20620712f7cccb659cd0e9d073d1991c038eb9fc57589", "https://bcr.bazel.build/modules/abseil-cpp/20211102.0/source.json": "7e3a9adf473e9af076ae485ed649d5641ad50ec5c11718103f34de03170d94ad", "https://bcr.bazel.build/modules/apple_support/1.5.0/MODULE.bazel": "50341a62efbc483e8a2a6aec30994a58749bd7b885e18dd96aa8c33031e558ef", "https://bcr.bazel.build/modules/apple_support/1.5.0/source.json": "eb98a7627c0bc486b57f598ad8da50f6625d974c8f723e9ea71bd39f709c9862", "https://bcr.bazel.build/modules/bazel_features/1.11.0/MODULE.bazel": "f9382337dd5a474c3b7d334c2f83e50b6eaedc284253334cf823044a26de03e8", "https://bcr.bazel.build/modules/bazel_features/1.11.0/source.json": "c9320aa53cd1c441d24bd6b716da087ad7e4ff0d9742a9884587596edfe53015", "https://bcr.bazel.build/modules/bazel_skylib/1.0.3/MODULE.bazel": "bcb0fd896384802d1ad283b4e4eb4d718eebd8cb820b0a2c3a347fb971afd9d8", "https://bcr.bazel.build/modules/bazel_skylib/1.2.1/MODULE.bazel": "f35baf9da0efe45fa3da1696ae906eea3d615ad41e2e3def4aeb4e8bc0ef9a7a", "https://bcr.bazel.build/modules/bazel_skylib/1.3.0/MODULE.bazel": "20228b92868bf5cfc41bda7afc8a8ba2a543201851de39d990ec957b513579c5", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/MODULE.bazel": "8fdee2dbaace6c252131c00e1de4b165dc65af02ea278476187765e1a617b917", "https://bcr.bazel.build/modules/bazel_skylib/1.6.1/source.json": "082ed5f9837901fada8c68c2f3ddc958bb22b6d654f71dd73f3df30d45d4b749", "https://bcr.bazel.build/modules/buildozer/7.1.2/MODULE.bazel": "2e8dd40ede9c454042645fd8d8d0cd1527966aa5c919de86661e62953cd73d84", "https://bcr.bazel.build/modules/buildozer/7.1.2/source.json": "c9028a501d2db85793a6996205c8de120944f50a0d570438fcae0457a5f9d1f8", "https://bcr.bazel.build/modules/googletest/1.11.0/MODULE.bazel": "3a83f095183f66345ca86aa13c58b59f9f94a2f81999c093d4eeaa2d262d12f4", "https://bcr.bazel.build/modules/googletest/1.11.0/source.json": "c73d9ef4268c91bd0c1cd88f1f9dfa08e814b1dbe89b5f594a9f08ba0244d206", "https://bcr.bazel.build/modules/platforms/0.0.4/MODULE.bazel": "9b328e31ee156f53f3c416a64f8491f7eb731742655a47c9eec4703a71644aee", "https://bcr.bazel.build/modules/platforms/0.0.5/MODULE.bazel": "5733b54ea419d5eaf7997054bb55f6a1d0b5ff8aedf0176fef9eea44f3acda37", "https://bcr.bazel.build/modules/platforms/0.0.6/MODULE.bazel": "ad6eeef431dc52aefd2d77ed20a4b353f8ebf0f4ecdd26a807d2da5aa8cd0615", "https://bcr.bazel.build/modules/platforms/0.0.7/MODULE.bazel": "72fd4a0ede9ee5c021f6a8dd92b503e089f46c227ba2813ff183b71616034814", "https://bcr.bazel.build/modules/platforms/0.0.8/MODULE.bazel": "9f142c03e348f6d263719f5074b21ef3adf0b139ee4c5133e2aa35664da9eb2d", "https://bcr.bazel.build/modules/platforms/0.0.9/MODULE.bazel": "4a87a60c927b56ddd67db50c89acaa62f4ce2a1d2149ccb63ffd871d5ce29ebc", "https://bcr.bazel.build/modules/platforms/0.0.9/source.json": "cd74d854bf16a9e002fb2ca7b1a421f4403cda29f824a765acd3a8c56f8d43e6", "https://bcr.bazel.build/modules/protobuf/21.7/MODULE.bazel": "a5a29bb89544f9b97edce05642fac225a808b5b7be74038ea3640fae2f8e66a7", "https://bcr.bazel.build/modules/protobuf/21.7/source.json": "bbe500720421e582ff2d18b0802464205138c06056f443184de39fbb8187b09b", "https://bcr.bazel.build/modules/protobuf/3.19.0/MODULE.bazel": "6b5fbb433f760a99a22b18b6850ed5784ef0e9928a72668b66e4d7ccd47db9b0", "https://bcr.bazel.build/modules/protobuf/3.19.6/MODULE.bazel": "9233edc5e1f2ee276a60de3eaa47ac4132302ef9643238f23128fea53ea12858", "https://bcr.bazel.build/modules/rules_cc/0.0.1/MODULE.bazel": "cb2aa0747f84c6c3a78dad4e2049c154f08ab9d166b1273835a8174940365647", "https://bcr.bazel.build/modules/rules_cc/0.0.2/MODULE.bazel": "6915987c90970493ab97393024c156ea8fb9f3bea953b2f3ec05c34f19b5695c", "https://bcr.bazel.build/modules/rules_cc/0.0.8/MODULE.bazel": "964c85c82cfeb6f3855e6a07054fdb159aced38e99a5eecf7bce9d53990afa3e", "https://bcr.bazel.build/modules/rules_cc/0.0.9/MODULE.bazel": "836e76439f354b89afe6a911a7adf59a6b2518fafb174483ad78a2a2fde7b1c5", "https://bcr.bazel.build/modules/rules_cc/0.0.9/source.json": "1f1ba6fea244b616de4a554a0f4983c91a9301640c8fe0dd1d410254115c8430", "https://bcr.bazel.build/modules/rules_java/4.0.0/MODULE.bazel": "5a78a7ae82cd1a33cef56dc578c7d2a46ed0dca12643ee45edbb8417899e6f74", "https://bcr.bazel.build/modules/rules_java/7.6.5/MODULE.bazel": "481164be5e02e4cab6e77a36927683263be56b7e36fef918b458d7a8a1ebadb1", "https://bcr.bazel.build/modules/rules_java/7.6.5/source.json": "a805b889531d1690e3c72a7a7e47a870d00323186a9904b36af83aa3d053ee8d", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/MODULE.bazel": "a56b85e418c83eb1839819f0b515c431010160383306d13ec21959ac412d2fe7", "https://bcr.bazel.build/modules/rules_jvm_external/4.4.2/source.json": "a075731e1b46bc8425098512d038d416e966ab19684a10a34f4741295642fc35", "https://bcr.bazel.build/modules/rules_license/0.0.3/MODULE.bazel": "627e9ab0247f7d1e05736b59dbb1b6871373de5ad31c3011880b4133cafd4bd0", "https://bcr.bazel.build/modules/rules_license/0.0.7/MODULE.bazel": "088fbeb0b6a419005b89cf93fe62d9517c0a2b8bb56af3244af65ecfe37e7d5d", "https://bcr.bazel.build/modules/rules_license/0.0.7/source.json": "355cc5737a0f294e560d52b1b7a6492d4fff2caf0bef1a315df5a298fca2d34a", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/MODULE.bazel": "df99f03fc7934a4737122518bb87e667e62d780b610910f0447665a7e2be62dc", "https://bcr.bazel.build/modules/rules_pkg/0.7.0/source.json": "c2557066e0c0342223ba592510ad3d812d4963b9024831f7f66fd0584dd8c66c", "https://bcr.bazel.build/modules/rules_proto/4.0.0/MODULE.bazel": "a7a7b6ce9bee418c1a760b3d84f83a299ad6952f9903c67f19e4edd964894e06", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/MODULE.bazel": "e8dff86b0971688790ae75528fe1813f71809b5afd57facb44dad9e8eca631b7", "https://bcr.bazel.build/modules/rules_proto/5.3.0-21.7/source.json": "d57902c052424dfda0e71646cb12668d39c4620ee0544294d9d941e7d12bc3a9", "https://bcr.bazel.build/modules/rules_python/0.10.2/MODULE.bazel": "cc82bc96f2997baa545ab3ce73f196d040ffb8756fd2d66125a530031cd90e5f", "https://bcr.bazel.build/modules/rules_python/0.22.1/MODULE.bazel": "26114f0c0b5e93018c0c066d6673f1a2c3737c7e90af95eff30cfee38d0bbac7", "https://bcr.bazel.build/modules/rules_python/0.22.1/source.json": "57226905e783bae7c37c2dd662be078728e48fa28ee4324a7eabcafb5a43d014", "https://bcr.bazel.build/modules/rules_python/0.4.0/MODULE.bazel": "9208ee05fd48bf09ac60ed269791cf17fb343db56c8226a720fbb1cdf467166c", "https://bcr.bazel.build/modules/stardoc/0.5.1/MODULE.bazel": "1a05d92974d0c122f5ccf09291442580317cdd859f07a8655f1db9a60374f9f8", "https://bcr.bazel.build/modules/stardoc/0.5.1/source.json": "a96f95e02123320aa015b956f29c00cb818fa891ef823d55148e1a362caacf29", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/MODULE.bazel": "7298990c00040a0e2f121f6c32544bab27d4452f80d9ce51349b1a28f3005c43", "https://bcr.bazel.build/modules/upb/0.0.0-20220923-a547704/source.json": "f1ef7d3f9e0e26d4b23d1c39b5f5de71f584dd7d1b4ef83d9bbba6ec7a6a6459", "https://bcr.bazel.build/modules/zlib/1.2.11/MODULE.bazel": "07b389abc85fdbca459b69e2ec656ae5622873af3f845e1c9d80fe179f3effa0", "https://bcr.bazel.build/modules/zlib/1.2.12/MODULE.bazel": "3b1a8834ada2a883674be8cbd36ede1b6ec481477ada359cd2d3ddc562340b27", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/MODULE.bazel": "af322bc08976524477c79d1e45e241b6efbeb918c497e8840b8ab116802dda79", "https://bcr.bazel.build/modules/zlib/1.3.1.bcr.3/source.json": "2be409ac3c7601245958cd4fcdff4288be79ed23bd690b4b951f500d54ee6e7d" }, "selectedYankedVersions": {}, "moduleExtensions": { "@@apple_support~//crosstool:setup.bzl%apple_cc_configure_extension": { "general": { "bzlTransitiveDigest": "PjIds3feoYE8SGbbIq2SFTZy3zmxeO2tQevJZNDo7iY=", "usagesDigest": "+hz7IHWN6A1oVJJWNDB6yZRG+RYhF76wAYItpAeIUIg=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { "local_config_apple_cc_toolchains": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", "ruleClassName": "_apple_cc_autoconf_toolchains", "attributes": {} }, "local_config_apple_cc": { "bzlFile": "@@apple_support~//crosstool:setup.bzl", "ruleClassName": "_apple_cc_autoconf", "attributes": {} } }, "recordedRepoMappingEntries": [ [ "apple_support~", "bazel_tools", "bazel_tools" ] ] } }, "@@platforms//host:extension.bzl%host_platform": { "general": { "bzlTransitiveDigest": "xelQcPZH8+tmuOHVjL9vDxMnnQNMlwj0SlvgoqBkm4U=", "usagesDigest": "pCYpDQmqMbmiiPI1p2Kd3VLm5T48rRAht5WdW0X2GlA=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { "host_platform": { "bzlFile": "@@platforms//host:extension.bzl", "ruleClassName": "host_platform_repo", "attributes": {} } }, "recordedRepoMappingEntries": [] } } } } codspeed-4.4.1/instrument-hooks/example/instrument-hooks/includes/callgrind.h000064400000000000000000000130471046102023000256600ustar 00000000000000/* ---------------------------------------------------------------- Notice that the following BSD-style license applies to this one file (callgrind.h) only. The rest of Valgrind is licensed under the terms of the GNU General Public License, version 2, unless otherwise indicated. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- This file is part of callgrind, a valgrind tool for cache simulation and call tree tracing. Copyright (C) 2003-2017 Josef Weidendorfer. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------- Notice that the above BSD-style license applies to this one file (callgrind.h) only. The entire rest of Valgrind is licensed under the terms of the GNU General Public License, version 2. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- */ #ifndef __CALLGRIND_H #define __CALLGRIND_H #include "valgrind.h" /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! This enum comprises an ABI exported by Valgrind to programs which use client requests. DO NOT CHANGE THE ORDER OF THESE ENTRIES, NOR DELETE ANY -- add new ones at the end. The identification ('C','T') for Callgrind has historical reasons: it was called "Calltree" before. Besides, ('C','G') would clash with cachegrind. */ typedef enum { VG_USERREQ__DUMP_STATS = VG_USERREQ_TOOL_BASE('C', 'T'), VG_USERREQ__ZERO_STATS, VG_USERREQ__TOGGLE_COLLECT, VG_USERREQ__DUMP_STATS_AT, VG_USERREQ__START_INSTRUMENTATION, VG_USERREQ__STOP_INSTRUMENTATION } Vg_CallgrindClientRequest; /* Dump current state of cost centers, and zero them afterwards */ #define CALLGRIND_DUMP_STATS \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS, 0, 0, 0, 0, 0) /* Dump current state of cost centers, and zero them afterwards. The argument is appended to a string stating the reason which triggered the dump. This string is written as a description field into the profile data dump. */ #define CALLGRIND_DUMP_STATS_AT(pos_str) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS_AT, pos_str, 0, 0, 0, \ 0) /* Zero cost centers */ #define CALLGRIND_ZERO_STATS \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__ZERO_STATS, 0, 0, 0, 0, 0) /* Toggles collection state. The collection state specifies whether the happening of events should be noted or if they are to be ignored. Events are noted by increment of counters in a cost center */ #define CALLGRIND_TOGGLE_COLLECT \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__TOGGLE_COLLECT, 0, 0, 0, 0, 0) /* Start full callgrind instrumentation if not already switched on. When cache simulation is done, it will flush the simulated cache; this will lead to an artificial cache warmup phase afterwards with cache misses which would not have happened in reality. */ #define CALLGRIND_START_INSTRUMENTATION \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__START_INSTRUMENTATION, 0, 0, 0, \ 0, 0) /* Stop full callgrind instrumentation if not already switched off. This flushes Valgrinds translation cache, and does no additional instrumentation afterwards, which effectivly will run at the same speed as the "none" tool (ie. at minimal slowdown). Use this to bypass Callgrind aggregation for uninteresting code parts. To start Callgrind in this mode to ignore the setup phase, use the option "--instr-atstart=no". */ #define CALLGRIND_STOP_INSTRUMENTATION \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STOP_INSTRUMENTATION, 0, 0, 0, \ 0, 0) #endif /* __CALLGRIND_H */ codspeed-4.4.1/instrument-hooks/example/instrument-hooks/includes/compat.h000064400000000000000000000020201046102023000251710ustar 00000000000000// Compatibility shims for cross-platform compilation // This header provides compatibility macros for different libc implementations #ifndef COMPAT_H #define COMPAT_H // Detect musl libc // Musl doesn't define a specific macro, but we can detect it by checking for // the absence of __GLIBC__ on Linux systems, or by checking features.h #if defined(__linux__) && !defined(__GLIBC__) && !defined(__BIONIC__) #define COMPAT_MUSL 1 #endif // Musl libc compatibility // Musl doesn't have separate *64 functions because all functions are 64-bit // clean. Map the glibc LFS64 (Large File Support) functions to their standard // equivalents. #if defined(COMPAT_MUSL) #define openat64 openat #define fopen64 fopen #define freopen64 freopen #define lseek64 lseek #define fseeko64 fseeko #define ftello64 ftello #define fgetpos64 fgetpos #define fsetpos64 fsetpos #define stat64 stat #define fstat64 fstat #define lstat64 lstat #define fstatat64 fstatat #define mmap64 mmap #define off64_t off_t #define ino64_t ino_t #endif #endif // COMPAT_H codspeed-4.4.1/instrument-hooks/example/instrument-hooks/includes/core.h000064400000000000000000000051401046102023000246440ustar 00000000000000// This file was manually created and exposes the functions of this library. // TODO: Can we automatically generate this file? #ifndef CORE_H #define CORE_H #include #include #include #ifdef __cplusplus extern "C" { #endif #ifndef _WIN32 #include "callgrind.h" #include "valgrind.h" #else #define CALLGRIND_START_INSTRUMENTATION #define CALLGRIND_STOP_INSTRUMENTATION #define CALLGRIND_ZERO_STATS #endif typedef uint64_t *InstrumentHooks; InstrumentHooks *instrument_hooks_init(void); void instrument_hooks_deinit(InstrumentHooks *); bool instrument_hooks_is_instrumented(InstrumentHooks *); uint8_t instrument_hooks_start_benchmark(InstrumentHooks *); uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *); uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); // Deprecated: use instrument_hooks_set_executed_benchmark instead uint8_t instrument_hooks_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name, const char *version); #define MARKER_TYPE_SAMPLE_START 0 #define MARKER_TYPE_SAMPLE_END 1 #define MARKER_TYPE_BENCHMARK_START 2 #define MARKER_TYPE_BENCHMARK_END 3 uint8_t instrument_hooks_add_marker(InstrumentHooks *, uint32_t pid, uint8_t marker_type, uint64_t timestamp); uint64_t instrument_hooks_current_timestamp(void); void callgrind_start_instrumentation(); void callgrind_stop_instrumentation(); // Feature flags for instrument hooks typedef enum { FEATURE_DISABLE_CALLGRIND_MARKERS = 0, } instrument_hooks_feature_t; void instrument_hooks_set_feature(instrument_hooks_feature_t feature, bool enabled); // Header functions that will be inlined. This can be used by languages that // directly consume the headers such as C or C++. This will allow for more // precise tracking of the benchmark performance. static inline uint8_t instrument_hooks_start_benchmark_inline( InstrumentHooks *instance) { instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); if (instrument_hooks_start_benchmark(instance) != 0) { return 1; } CALLGRIND_ZERO_STATS; CALLGRIND_START_INSTRUMENTATION; return 0; } static inline uint8_t instrument_hooks_stop_benchmark_inline( InstrumentHooks *instance) { CALLGRIND_STOP_INSTRUMENTATION; return instrument_hooks_stop_benchmark(instance); } #ifdef __cplusplus } #endif #endif codspeed-4.4.1/instrument-hooks/example/instrument-hooks/includes/valgrind.h000064400000000000000000014725571046102023000255470ustar 00000000000000/* -*- c -*- ---------------------------------------------------------------- Notice that the following BSD-style license applies to this one file (valgrind.h) only. The rest of Valgrind is licensed under the terms of the GNU General Public License, version 2, unless otherwise indicated. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- This file is part of Valgrind, a dynamic binary instrumentation framework. Copyright (C) 2000-2017 Julian Seward. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------- Notice that the above BSD-style license applies to this one file (valgrind.h) only. The entire rest of Valgrind is licensed under the terms of the GNU General Public License, version 2. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- */ /* This file is for inclusion into client (your!) code. You can use these macros to manipulate and query Valgrind's execution inside your own programs. The resulting executables will still run without Valgrind, just a little bit more slowly than they otherwise would, but otherwise unchanged. When not running on valgrind, each client request consumes very few (eg. 7) instructions, so the resulting performance loss is negligible unless you plan to execute client requests millions of times per second. Nevertheless, if that is still a problem, you can compile with the NVALGRIND symbol defined (gcc -DNVALGRIND) so that client requests are not even compiled in. */ #ifndef __VALGRIND_H #define __VALGRIND_H /* ------------------------------------------------------------------ */ /* VERSION NUMBER OF VALGRIND */ /* ------------------------------------------------------------------ */ /* Specify Valgrind's version number, so that user code can conditionally compile based on our version number. Note that these were introduced at version 3.6 and so do not exist in version 3.5 or earlier. The recommended way to use them to check for "version X.Y or later" is (eg) #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \ && (__VALGRIND_MAJOR__ > 3 \ || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6)) */ #define __VALGRIND_MAJOR__ 3 #define __VALGRIND_MINOR__ 24 #include /* Nb: this file might be included in a file compiled with -ansi. So we can't use C++ style "//" comments nor the "asm" keyword (instead use "__asm__"). */ /* Derive some tags indicating what the target platform is. Note that in this file we're using the compiler's CPP symbols for identifying architectures, which are different to the ones we use within the rest of Valgrind. Note, __powerpc__ is active for both 32 and 64-bit PPC, whereas __powerpc64__ is only active for the latter (on Linux, that is). Misc note: how to find out what's predefined in gcc by default: gcc -Wp,-dM somefile.c */ #undef PLAT_x86_darwin #undef PLAT_amd64_darwin #undef PLAT_x86_freebsd #undef PLAT_amd64_freebsd #undef PLAT_arm64_freebsd #undef PLAT_x86_win32 #undef PLAT_amd64_win64 #undef PLAT_x86_linux #undef PLAT_amd64_linux #undef PLAT_ppc32_linux #undef PLAT_ppc64be_linux #undef PLAT_ppc64le_linux #undef PLAT_arm_linux #undef PLAT_arm64_linux #undef PLAT_s390x_linux #undef PLAT_mips32_linux #undef PLAT_mips64_linux #undef PLAT_nanomips_linux #undef PLAT_x86_solaris #undef PLAT_amd64_solaris #if defined(__APPLE__) && defined(__i386__) # define PLAT_x86_darwin 1 #elif defined(__APPLE__) && defined(__x86_64__) # define PLAT_amd64_darwin 1 #elif defined(__FreeBSD__) && defined(__i386__) # define PLAT_x86_freebsd 1 #elif defined(__FreeBSD__) && defined(__amd64__) # define PLAT_amd64_freebsd 1 #elif defined(__FreeBSD__) && defined(__aarch64__) && !defined(__arm__) # define PLAT_arm64_freebsd 1 #elif (defined(__MINGW32__) && defined(__i386__)) \ || defined(__CYGWIN32__) \ || (defined(_WIN32) && defined(_M_IX86)) # define PLAT_x86_win32 1 #elif (defined(__MINGW32__) && defined(__x86_64__)) \ || (defined(_WIN32) && defined(_M_X64)) /* __MINGW32__ and _WIN32 are defined in 64 bit mode as well. */ # define PLAT_amd64_win64 1 #elif defined(__linux__) && defined(__i386__) # define PLAT_x86_linux 1 #elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__) # define PLAT_amd64_linux 1 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__) # define PLAT_ppc32_linux 1 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2 /* Big Endian uses ELF version 1 */ # define PLAT_ppc64be_linux 1 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2 /* Little Endian uses ELF version 2 */ # define PLAT_ppc64le_linux 1 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__) # define PLAT_arm_linux 1 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__) # define PLAT_arm64_linux 1 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__) # define PLAT_s390x_linux 1 #elif defined(__linux__) && defined(__mips__) && (__mips==64) # define PLAT_mips64_linux 1 #elif defined(__linux__) && defined(__mips__) && (__mips==32) # define PLAT_mips32_linux 1 #elif defined(__linux__) && defined(__nanomips__) # define PLAT_nanomips_linux 1 #elif defined(__sun) && defined(__i386__) # define PLAT_x86_solaris 1 #elif defined(__sun) && defined(__x86_64__) # define PLAT_amd64_solaris 1 #else /* If we're not compiling for our target platform, don't generate any inline asms. */ # if !defined(NVALGRIND) # define NVALGRIND 1 # endif #endif /* ------------------------------------------------------------------ */ /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS. There is nothing */ /* in here of use to end-users -- skip to the next section. */ /* ------------------------------------------------------------------ */ /* * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client * request. Accepts both pointers and integers as arguments. * * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind * client request that does not return a value. * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind * client request and whose value equals the client request result. Accepts * both pointers and integers as arguments. Note that such calls are not * necessarily pure functions -- they may have side effects. */ #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, \ _zzq_request, _zzq_arg1, _zzq_arg2, \ _zzq_arg3, _zzq_arg4, _zzq_arg5) \ do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default), \ (_zzq_request), (_zzq_arg1), (_zzq_arg2), \ (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0) #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1, \ _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ (_zzq_request), (_zzq_arg1), (_zzq_arg2), \ (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0) #if defined(NVALGRIND) /* Define NVALGRIND to completely remove the Valgrind magic sequence from the compiled code (analogous to NDEBUG's effects on assert()) */ #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ (_zzq_default) #else /* ! NVALGRIND */ /* The following defines the magic code sequences which the JITter spots and handles magically. Don't look too closely at them as they will rot your brain. The assembly code sequences for all architectures is in this one file. This is because this file must be stand-alone, and we don't want to have multiple files. For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default value gets put in the return slot, so that everything works when this is executed not under Valgrind. Args are passed in a memory block, and so there's no intrinsic limit to the number that could be passed, but it's currently five. The macro args are: _zzq_rlval result lvalue _zzq_default default value (result returned when running on real CPU) _zzq_request request code _zzq_arg1..5 request params The other two macros are used to support function wrapping, and are a lot simpler. VALGRIND_GET_NR_CONTEXT returns the value of the guest's NRADDR pseudo-register and whatever other information is needed to safely run the call original from the wrapper: on ppc64-linux, the R2 value at the divert point is also needed. This information is abstracted into a user-visible type, OrigFn. VALGRIND_CALL_NOREDIR_* behaves the same as the following on the guest, but guarantees that the branch instruction will not be redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64: branch-and-link-to-r11. VALGRIND_CALL_NOREDIR is just text, not a complete inline asm, since it needs to be combined with more magic inline asm stuff to be useful. */ /* ----------------- x86-{linux,darwin,solaris} ---------------- */ #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin) \ || (defined(PLAT_x86_win32) && defined(__GNUC__)) \ || defined(PLAT_x86_solaris) || defined(PLAT_x86_freebsd) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "roll $3, %%edi ; roll $13, %%edi\n\t" \ "roll $29, %%edi ; roll $19, %%edi\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %EDX = client_request ( %EAX ) */ \ "xchgl %%ebx,%%ebx" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), "0" (_zzq_default) \ : "cc", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %EAX = guest_NRADDR */ \ "xchgl %%ecx,%%ecx" \ : "=a" (__addr) \ : \ : "cc", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_EAX \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%EAX */ \ "xchgl %%edx,%%edx\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "xchgl %%edi,%%edi\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) || PLAT_x86_solaris */ /* ------------------------- x86-Win32 ------------------------- */ #if defined(PLAT_x86_win32) && !defined(__GNUC__) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #if defined(_MSC_VER) #define __SPECIAL_INSTRUCTION_PREAMBLE \ __asm rol edi, 3 __asm rol edi, 13 \ __asm rol edi, 29 __asm rol edi, 19 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ valgrind_do_client_request_expr((uintptr_t)(_zzq_default), \ (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1), \ (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3), \ (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5)) static __inline uintptr_t valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request, uintptr_t _zzq_arg1, uintptr_t _zzq_arg2, uintptr_t _zzq_arg3, uintptr_t _zzq_arg4, uintptr_t _zzq_arg5) { volatile uintptr_t _zzq_args[6]; volatile unsigned int _zzq_result; _zzq_args[0] = (uintptr_t)(_zzq_request); _zzq_args[1] = (uintptr_t)(_zzq_arg1); _zzq_args[2] = (uintptr_t)(_zzq_arg2); _zzq_args[3] = (uintptr_t)(_zzq_arg3); _zzq_args[4] = (uintptr_t)(_zzq_arg4); _zzq_args[5] = (uintptr_t)(_zzq_arg5); __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default __SPECIAL_INSTRUCTION_PREAMBLE /* %EDX = client_request ( %EAX ) */ __asm xchg ebx,ebx __asm mov _zzq_result, edx } return _zzq_result; } #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm { __SPECIAL_INSTRUCTION_PREAMBLE \ /* %EAX = guest_NRADDR */ \ __asm xchg ecx,ecx \ __asm mov __addr, eax \ } \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_EAX ERROR #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm { __SPECIAL_INSTRUCTION_PREAMBLE \ __asm xchg edi,edi \ } \ } while (0) #else #error Unsupported compiler. #endif #endif /* PLAT_x86_win32 */ /* ----------------- amd64-{linux,darwin,solaris} --------------- */ #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin) \ || defined(PLAT_amd64_solaris) \ || defined(PLAT_amd64_freebsd) \ || (defined(PLAT_amd64_win64) && defined(__GNUC__)) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \ "rolq $61, %%rdi ; rolq $51, %%rdi\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %RDX = client_request ( %RAX ) */ \ "xchgq %%rbx,%%rbx" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), "0" (_zzq_default) \ : "cc", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %RAX = guest_NRADDR */ \ "xchgq %%rcx,%%rcx" \ : "=a" (__addr) \ : \ : "cc", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_RAX \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%RAX */ \ "xchgq %%rdx,%%rdx\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "xchgq %%rdi,%%rdi\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */ /* ------------------------- amd64-Win64 ------------------------- */ #if defined(PLAT_amd64_win64) && !defined(__GNUC__) #error Unsupported compiler. #endif /* PLAT_amd64_win64 */ /* ------------------------ ppc32-linux ------------------------ */ #if defined(PLAT_ppc32_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rlwinm 0,0,3,0,31 ; rlwinm 0,0,13,0,31\n\t" \ "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned int _zzq_args[6]; \ unsigned int _zzq_result; \ unsigned int* _zzq_ptr; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R11 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc32_linux */ /* ------------------------ ppc64-linux ------------------------ */ #if defined(PLAT_ppc64be_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ unsigned long int r2; /* what tocptr do we need? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ "rotldi 0,0,61 ; rotldi 0,0,51\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned long int _zzq_args[6]; \ unsigned long int _zzq_result; \ unsigned long int* _zzq_ptr; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR_GPR2 */ \ "or 4,4,4\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->r2 = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R11 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc64be_linux */ #if defined(PLAT_ppc64le_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ unsigned long int r2; /* what tocptr do we need? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ "rotldi 0,0,61 ; rotldi 0,0,51\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned long int _zzq_args[6]; \ unsigned long int _zzq_result; \ unsigned long int* _zzq_ptr; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR_GPR2 */ \ "or 4,4,4\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->r2 = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R12 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc64le_linux */ /* ------------------------- arm-linux ------------------------- */ #if defined(PLAT_arm_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "mov r12, r12, ror #3 ; mov r12, r12, ror #13 \n\t" \ "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("mov r3, %1\n\t" /*default*/ \ "mov r4, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* R3 = client_request ( R4 ) */ \ "orr r10, r10, r10\n\t" \ "mov %0, r3" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "cc","memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* R3 = guest_NRADDR */ \ "orr r11, r11, r11\n\t" \ "mov %0, r3" \ : "=r" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R4 */ \ "orr r12, r12, r12\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "orr r9, r9, r9\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_arm_linux */ /* ------------------------ arm64-{linux,freebsd} ------------------------- */ #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "ror x12, x12, #3 ; ror x12, x12, #13 \n\t" \ "ror x12, x12, #51 ; ror x12, x12, #61 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile("mov x3, %1\n\t" /*default*/ \ "mov x4, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* X3 = client_request ( X4 ) */ \ "orr x10, x10, x10\n\t" \ "mov %0, x3" /*result*/ \ : "=r" (_zzq_result) \ : "r" ((unsigned long int)(_zzq_default)), \ "r" (&_zzq_args[0]) \ : "cc","memory", "x3", "x4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* X3 = guest_NRADDR */ \ "orr x11, x11, x11\n\t" \ "mov %0, x3" \ : "=r" (__addr) \ : \ : "cc", "memory", "x3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir X8 */ \ "orr x12, x12, x12\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "orr x9, x9, x9\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_arm64_linux || PLAT_arm64_freebsd */ /* ------------------------ s390x-linux ------------------------ */ #if defined(PLAT_s390x_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific * code. This detection is implemented in platform specific toIR.c * (e.g. VEX/priv/guest_s390_decoder.c). */ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "lr 15,15\n\t" \ "lr 1,1\n\t" \ "lr 2,2\n\t" \ "lr 3,3\n\t" #define __CLIENT_REQUEST_CODE "lr 2,2\n\t" #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t" #define __CALL_NO_REDIR_CODE "lr 4,4\n\t" #define __VEX_INJECT_IR_CODE "lr 5,5\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile(/* r2 = args */ \ "lgr 2,%1\n\t" \ /* r3 = default */ \ "lgr 3,%2\n\t" \ __SPECIAL_INSTRUCTION_PREAMBLE \ __CLIENT_REQUEST_CODE \ /* results = r3 */ \ "lgr %0, 3\n\t" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), \ "0" ((unsigned long int)_zzq_default) \ : "cc", "2", "3", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ __GET_NR_CONTEXT_CODE \ "lgr %0, 3\n\t" \ : "=a" (__addr) \ : \ : "cc", "3", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_R1 \ __SPECIAL_INSTRUCTION_PREAMBLE \ __CALL_NO_REDIR_CODE #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ __VEX_INJECT_IR_CODE); \ } while (0) #endif /* PLAT_s390x_linux */ /* ------------------------- mips32-linux ---------------- */ #if defined(PLAT_mips32_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; /* .word 0x342 * .word 0x742 * .word 0xC2 * .word 0x4C2*/ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "srl $0, $0, 13\n\t" \ "srl $0, $0, 29\n\t" \ "srl $0, $0, 3\n\t" \ "srl $0, $0, 19\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("move $11, %1\n\t" /*default*/ \ "move $12, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* T3 = client_request ( T4 ) */ \ "or $13, $13, $13\n\t" \ "move %0, $11\n\t" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$11", "$12", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %t9 = guest_NRADDR */ \ "or $14, $14, $14\n\t" \ "move %0, $11" /*result*/ \ : "=r" (__addr) \ : \ : "$11" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%t9 */ \ "or $15, $15, $15\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or $11, $11, $11\n\t" \ ); \ } while (0) #endif /* PLAT_mips32_linux */ /* ------------------------- mips64-linux ---------------- */ #if defined(PLAT_mips64_linux) typedef struct { unsigned long nraddr; /* where's the code? */ } OrigFn; /* dsll $0,$0, 3 * dsll $0,$0, 13 * dsll $0,$0, 29 * dsll $0,$0, 19*/ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "dsll $0,$0, 3 ; dsll $0,$0,13\n\t" \ "dsll $0,$0,29 ; dsll $0,$0,19\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile("move $11, %1\n\t" /*default*/ \ "move $12, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* $11 = client_request ( $12 ) */ \ "or $13, $13, $13\n\t" \ "move %0, $11\n\t" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$11", "$12", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* $11 = guest_NRADDR */ \ "or $14, $14, $14\n\t" \ "move %0, $11" /*result*/ \ : "=r" (__addr) \ : \ : "$11"); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir $25 */ \ "or $15, $15, $15\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or $11, $11, $11\n\t" \ ); \ } while (0) #endif /* PLAT_mips64_linux */ #if defined(PLAT_nanomips_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; /* 8000 c04d srl zero, zero, 13 8000 c05d srl zero, zero, 29 8000 c043 srl zero, zero, 3 8000 c053 srl zero, zero, 19 */ #define __SPECIAL_INSTRUCTION_PREAMBLE "srl[32] $zero, $zero, 13 \n\t" \ "srl[32] $zero, $zero, 29 \n\t" \ "srl[32] $zero, $zero, 3 \n\t" \ "srl[32] $zero, $zero, 19 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("move $a7, %1\n\t" /* default */ \ "move $t0, %2\n\t" /* ptr */ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* $a7 = client_request( $t0 ) */ \ "or[32] $t0, $t0, $t0\n\t" \ "move %0, $a7\n\t" /* result */ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$a7", "$t0", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* $a7 = guest_NRADDR */ \ "or[32] $t1, $t1, $t1\n\t" \ "move %0, $a7" /*result*/ \ : "=r" (__addr) \ : \ : "$a7"); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir $25 */ \ "or[32] $t2, $t2, $t2\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or[32] $t3, $t3, $t3\n\t" \ ); \ } while (0) #endif /* Insert assembly code for other platforms here... */ #endif /* NVALGRIND */ /* ------------------------------------------------------------------ */ /* PLATFORM SPECIFICS for FUNCTION WRAPPING. This is all very */ /* ugly. It's the least-worst tradeoff I can think of. */ /* ------------------------------------------------------------------ */ /* This section defines magic (a.k.a appalling-hack) macros for doing guaranteed-no-redirection macros, so as to get from function wrappers to the functions they are wrapping. The whole point is to construct standard call sequences, but to do the call itself with a special no-redirect call pseudo-instruction that the JIT understands and handles specially. This section is long and repetitious, and I can't see a way to make it shorter. The naming scheme is as follows: CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc} 'W' stands for "word" and 'v' for "void". Hence there are different macros for calling arity 0, 1, 2, 3, 4, etc, functions, and for each, the possibility of returning a word-typed result, or no result. */ /* Use these to write the name of your wrapper. NOTE: duplicates VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. NOTE also: inserts the default behaviour equivalance class tag "0000" into the name. See pub_tool_redir.h for details -- normally you don't need to think about this, though. */ /* Use an extra level of macroisation so as to ensure the soname/fnname args are fully macro-expanded before pasting them together. */ #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname) \ VG_CONCAT4(_vgw00000ZU_,soname,_,fnname) #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname) \ VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname) /* Use this macro from within a wrapper function to collect the context (address and possibly other info) of the original function. Once you have that you can then use it in one of the CALL_FN_ macros. The type of the argument _lval is OrigFn. */ #define VALGRIND_GET_ORIG_FN(_lval) VALGRIND_GET_NR_CONTEXT(_lval) /* Also provide end-user facilities for function replacement, rather than wrapping. A replacement function differs from a wrapper in that it has no way to get hold of the original function being called, and hence no way to call onwards to it. In a replacement function, VALGRIND_GET_ORIG_FN always returns zero. */ #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname) \ VG_CONCAT4(_vgr00000ZU_,soname,_,fnname) #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname) \ VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname) /* Derivatives of the main macros below, for calling functions returning void. */ #define CALL_FN_v_v(fnptr) \ do { volatile unsigned long _junk; \ CALL_FN_W_v(_junk,fnptr); } while (0) #define CALL_FN_v_W(fnptr, arg1) \ do { volatile unsigned long _junk; \ CALL_FN_W_W(_junk,fnptr,arg1); } while (0) #define CALL_FN_v_WW(fnptr, arg1,arg2) \ do { volatile unsigned long _junk; \ CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0) #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3) \ do { volatile unsigned long _junk; \ CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0) #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4) \ do { volatile unsigned long _junk; \ CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0) #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5) \ do { volatile unsigned long _junk; \ CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0) #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6) \ do { volatile unsigned long _junk; \ CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0) #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7) \ do { volatile unsigned long _junk; \ CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0) /* ----------------- x86-{linux,darwin,solaris} ---------------- */ #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin) \ || defined(PLAT_x86_solaris) || defined(PLAT_x86_freebsd) /* These regs are trashed by the hidden call. No need to mention eax as gcc can already see that, plus causes gcc to bomb. */ #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "movl %%esp,%%edi\n\t" \ "andl $0xfffffff0,%%esp\n\t" #define VALGRIND_RESTORE_STACK \ "movl %%edi,%%esp\n\t" /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 44(%%eax)\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 48(%%eax)\n\t" \ "pushl 44(%%eax)\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_x86_linux || PLAT_x86_darwin || PLAT_x86_solaris */ /* ---------------- amd64-{linux,darwin,solaris} --------------- */ #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin) \ || defined(PLAT_amd64_solaris) || defined(PLAT_amd64_freebsd) /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi", \ "rdi", "r8", "r9", "r10", "r11" /* This is all pretty complex. It's so as to make stack unwinding work reliably. See bug 243270. The basic problem is the sub and add of 128 of %rsp in all of the following macros. If gcc believes the CFA is in %rsp, then unwinding may fail, because what's at the CFA is not what gcc "expected" when it constructs the CFIs for the places where the macros are instantiated. But we can't just add a CFI annotation to increase the CFA offset by 128, to match the sub of 128 from %rsp, because we don't know whether gcc has chosen %rsp as the CFA at that point, or whether it has chosen some other register (eg, %rbp). In the latter case, adding a CFI annotation to change the CFA offset is simply wrong. So the solution is to get hold of the CFA using __builtin_dwarf_cfa(), put it in a known register, and add a CFI annotation to say what the register is. We choose %rbp for this (perhaps perversely), because: (1) %rbp is already subject to unwinding. If a new register was chosen then the unwinder would have to unwind it in all stack traces, which is expensive, and (2) %rbp is already subject to precise exception updates in the JIT. If a new register was chosen, we'd have to have precise exceptions for it too, which reduces performance of the generated code. However .. one extra complication. We can't just whack the result of __builtin_dwarf_cfa() into %rbp and then add %rbp to the list of trashed registers at the end of the inline assembly fragments; gcc won't allow %rbp to appear in that list. Hence instead we need to stash %rbp in %r15 for the duration of the asm, and say that %r15 is trashed instead. gcc seems happy to go with that. Oh .. and this all needs to be conditionalised so that it is unchanged from before this commit, when compiled with older gccs that don't support __builtin_dwarf_cfa. Furthermore, since this header file is freestanding, it has to be independent of config.h, and so the following conditionalisation cannot depend on configure time checks. Although it's not clear from 'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)', this expression excludes Darwin. .cfi directives in Darwin assembly appear to be completely different and I haven't investigated how they work. For even more entertainment value, note we have to use the completely undocumented __builtin_dwarf_cfa(), which appears to really compute the CFA, whereas __builtin_frame_address(0) claims to but actually doesn't. See https://bugs.kde.org/show_bug.cgi?id=243270#c47 */ #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM) # define __FRAME_POINTER \ ,"r"(__builtin_dwarf_cfa()) # define VALGRIND_CFI_PROLOGUE \ "movq %%rbp, %%r15\n\t" \ "movq %2, %%rbp\n\t" \ ".cfi_remember_state\n\t" \ ".cfi_def_cfa rbp, 0\n\t" # define VALGRIND_CFI_EPILOGUE \ "movq %%r15, %%rbp\n\t" \ ".cfi_restore_state\n\t" #else # define __FRAME_POINTER # define VALGRIND_CFI_PROLOGUE # define VALGRIND_CFI_EPILOGUE #endif /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "movq %%rsp,%%r14\n\t" \ "andq $0xfffffffffffffff0,%%rsp\n\t" #define VALGRIND_RESTORE_STACK \ "movq %%r14,%%rsp\n\t" /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned long) == 8. */ /* NB 9 Sept 07. There is a nasty kludge here in all these CALL_FN_ macros. In order not to trash the stack redzone, we need to drop %rsp by 128 before the hidden call, and restore afterwards. The nastyness is that it is only by luck that the stack still appears to be unwindable during the hidden call - since then the behaviour of any routine using this macro does not match what the CFI data says. Sigh. Why is this important? Imagine that a wrapper has a stack allocated local, and passes to the hidden call, a pointer to it. Because gcc does not know about the hidden call, it may allocate that local in the redzone. Unfortunately the hidden call may then trash it before it comes to use it. So we must step clear of the redzone, for the duration of the hidden call, to make it safe. Probably the same problem afflicts the other redzone-style ABIs too (ppc64-linux); but for those, the stack is self describing (none of this CFI nonsense) so at least messing with the stack pointer doesn't give a danger of non-unwindable stack. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 88(%%rax)\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 96(%%rax)\n\t" \ "pushq 88(%%rax)\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */ /* ------------------------ ppc32-linux ------------------------ */ #if defined(PLAT_ppc32_linux) /* This is useful for finding out about the on-stack stuff: extern int f9 ( int,int,int,int,int,int,int,int,int ); extern int f10 ( int,int,int,int,int,int,int,int,int,int ); extern int f11 ( int,int,int,int,int,int,int,int,int,int,int ); extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int ); int g9 ( void ) { return f9(11,22,33,44,55,66,77,88,99); } int g10 ( void ) { return f10(11,22,33,44,55,66,77,88,99,110); } int g11 ( void ) { return f11(11,22,33,44,55,66,77,88,99,110,121); } int g12 ( void ) { return f12(11,22,33,44,55,66,77,88,99,110,121,132); } */ /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rlwinm 1,1,0,0,27\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc32-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-16\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-16\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-32\n\t" \ /* arg11 */ \ "lwz 3,44(11)\n\t" \ "stw 3,16(1)\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ _argvec[12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-32\n\t" \ /* arg12 */ \ "lwz 3,48(11)\n\t" \ "stw 3,20(1)\n\t" \ /* arg11 */ \ "lwz 3,44(11)\n\t" \ "stw 3,16(1)\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc32_linux */ /* ------------------------ ppc64-linux ------------------------ */ #if defined(PLAT_ppc64be_linux) /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rldicr 1,1,0,59\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+0]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+1]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+2]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+3]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+4]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+5]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+6]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+7]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+8]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+9]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+10]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+11]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg11 */ \ "ld 3,88(11)\n\t" \ "std 3,128(1)\n\t" \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+12]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ _argvec[2+12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg12 */ \ "ld 3,96(11)\n\t" \ "std 3,136(1)\n\t" \ /* arg11 */ \ "ld 3,88(11)\n\t" \ "std 3,128(1)\n\t" \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc64be_linux */ /* ------------------------- ppc64le-linux ----------------------- */ #if defined(PLAT_ppc64le_linux) /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rldicr 1,1,0,59\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+0]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+1]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+2]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+3]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+4]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+5]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+6]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+7]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+8]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+9]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+10]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+11]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg11 */ \ "ld 3,88(12)\n\t" \ "std 3,112(1)\n\t" \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+12]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ _argvec[2+12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg12 */ \ "ld 3,96(12)\n\t" \ "std 3,120(1)\n\t" \ /* arg11 */ \ "ld 3,88(12)\n\t" \ "std 3,112(1)\n\t" \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc64le_linux */ /* ------------------------- arm-linux ------------------------- */ #if defined(PLAT_arm_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4", "r12", "r14" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ /* This is a bit tricky. We store the original stack pointer in r10 as it is callee-saves. gcc doesn't allow the use of r11 for some reason. Also, we can't directly "bic" the stack pointer in thumb mode since r13 isn't an allowed register number in that context. So use r4 as a temporary, since that is about to get trashed anyway, just after each use of this macro. Side effect is we need to be very careful about any future changes, since VALGRIND_ALIGN_STACK simply assumes r4 is usable. */ #define VALGRIND_ALIGN_STACK \ "mov r10, sp\n\t" \ "mov r4, sp\n\t" \ "bic r4, r4, #7\n\t" \ "mov sp, r4\n\t" #define VALGRIND_RESTORE_STACK \ "mov sp, r10\n\t" /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "push {r0} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "push {r0, r1} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "push {r0, r1, r2} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "push {r0, r1, r2, r3} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #40] \n\t" \ "push {r0} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #40] \n\t" \ "ldr r1, [%1, #44] \n\t" \ "push {r0, r1} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #40] \n\t" \ "ldr r1, [%1, #44] \n\t" \ "ldr r2, [%1, #48] \n\t" \ "push {r0, r1, r2} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_arm_linux */ /* ------------------------ arm64-linux ------------------------ */ #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9", \ "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", \ "x18", "x19", "x20", "x30", \ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", \ "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", \ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", \ "v26", "v27", "v28", "v29", "v30", "v31" /* x21 is callee-saved, so we can use it to save and restore SP around the hidden call. */ #define VALGRIND_ALIGN_STACK \ "mov x21, sp\n\t" \ "bic sp, x21, #15\n\t" #define VALGRIND_RESTORE_STACK \ "mov sp, x21\n\t" /* These CALL_FN_ macros assume that on arm64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x20 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x20 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x30 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1, #88] \n\t" \ "str x8, [sp, #16] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11, \ arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x30 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1, #88] \n\t" \ "str x8, [sp, #16] \n\t" \ "ldr x8, [%1, #96] \n\t" \ "str x8, [sp, #24] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_arm64_linux */ /* ------------------------- s390x-linux ------------------------- */ #if defined(PLAT_s390x_linux) /* Similar workaround as amd64 (see above), but we use r11 as frame pointer and save the old r11 in r7. r11 might be used for argvec, therefore we copy argvec in r1 since r1 is clobbered after the call anyway. */ #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM) # define __FRAME_POINTER \ ,"d"(__builtin_dwarf_cfa()) # define VALGRIND_CFI_PROLOGUE \ ".cfi_remember_state\n\t" \ "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */ \ "lgr 7,11\n\t" \ "lgr 11,%2\n\t" \ ".cfi_def_cfa 11, 0\n\t" # define VALGRIND_CFI_EPILOGUE \ "lgr 11, 7\n\t" \ ".cfi_restore_state\n\t" #else # define __FRAME_POINTER # define VALGRIND_CFI_PROLOGUE \ "lgr 1,%1\n\t" # define VALGRIND_CFI_EPILOGUE #endif /* Nb: On s390 the stack pointer is properly aligned *at all times* according to the s390 GCC maintainer. (The ABI specification is not precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and VALGRIND_RESTORE_STACK are not defined here. */ /* These regs are trashed by the hidden call. Note that we overwrite r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the function a proper return address. All others are ABI defined call clobbers. */ #if defined(__VX__) || defined(__S390_VX__) #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14", \ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", \ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", \ "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", \ "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" #else #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14", \ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" #endif /* Nb: Although r11 is modified in the asm snippets below (inside VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for two reasons: (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not modified (2) GCC will complain that r11 cannot appear inside a clobber section, when compiled with -O -fno-omit-frame-pointer */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 1, 0(1)\n\t" /* target->r1 */ \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "d" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) /* The call abi has the arguments in r2-r6 and stack */ #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1, arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-168\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,168\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-176\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,176\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-184\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,184\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-192\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,192\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-200\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,200\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10, arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-208\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "mvc 200(8,15), 88(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,208\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ _argvec[12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-216\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "mvc 200(8,15), 88(1)\n\t" \ "mvc 208(8,15), 96(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,216\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_s390x_linux */ /* ------------------------- mips32-linux ----------------------- */ #if defined(PLAT_mips32_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \ "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \ "$25", "$31" /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16\n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" /* arg1*/ \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 24\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 24 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 32\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "nop\n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 32 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 32\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 32 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 40\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 40 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 40\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 40 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 48\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 48 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 48\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 44(%1) \n\t" \ "sw $4, 40($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 48 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 56\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 44(%1) \n\t" \ "sw $4, 40($29) \n\t" \ "lw $4, 48(%1) \n\t" \ "sw $4, 44($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 56 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_mips32_linux */ /* ------------------------- nanomips-linux -------------------- */ #if defined(PLAT_nanomips_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$t4", "$t5", "$a0", "$a1", "$a2", \ "$a3", "$a4", "$a5", "$a6", "$a7", "$t0", "$t1", "$t2", "$t3", \ "$t8","$t9", "$at" /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ "lw $a6,28(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ "lw $a6,28(%1)\n\t" \ "lw $a7,32(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9,44(%1) \n\t" \ "sw $t9, 8($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9,44(%1) \n\t" \ "sw $t9, 8($sp) \n\t" \ "lw $t9,48(%1) \n\t" \ "sw $t9,12($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_nanomips_linux */ /* ------------------------- mips64-linux ------------------------- */ #if defined(PLAT_mips64_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \ "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \ "$25", "$31" /* These CALL_FN_ macros assume that on mips64-linux, sizeof(long long) == 8. */ #define MIPS64_LONG2REG_CAST(x) ((long long)(long)x) #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[1]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ __asm__ volatile( \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[2]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" /* arg1*/ \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[3]; \ volatile unsigned long long _res; \ _argvec[0] = _orig.nraddr; \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[4]; \ volatile unsigned long long _res; \ _argvec[0] = _orig.nraddr; \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[5]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[6]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[7]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[8]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[9]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[10]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ __asm__ volatile( \ "dsubu $29, $29, 8\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 8\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[11]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ __asm__ volatile( \ "dsubu $29, $29, 16\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 16\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[12]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ _argvec[11] = MIPS64_LONG2REG_CAST(arg11); \ __asm__ volatile( \ "dsubu $29, $29, 24\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 88(%1)\n\t" \ "sd $4, 16($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 24\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[13]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ _argvec[11] = MIPS64_LONG2REG_CAST(arg11); \ _argvec[12] = MIPS64_LONG2REG_CAST(arg12); \ __asm__ volatile( \ "dsubu $29, $29, 32\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 88(%1)\n\t" \ "sd $4, 16($29)\n\t" \ "ld $4, 96(%1)\n\t" \ "sd $4, 24($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 32\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #endif /* PLAT_mips64_linux */ /* ------------------------------------------------------------------ */ /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS. */ /* */ /* ------------------------------------------------------------------ */ /* Some request codes. There are many more of these, but most are not exposed to end-user view. These are the public ones, all of the form 0x1000 + small_number. Core ones are in the range 0x00000000--0x0000ffff. The non-public ones start at 0x2000. */ /* These macros are used by tools -- they must be public, but don't embed them into other programs. */ #define VG_USERREQ_TOOL_BASE(a,b) \ ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16)) #define VG_IS_TOOL_USERREQ(a, b, v) \ (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000)) /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! This enum comprises an ABI exported by Valgrind to programs which use client requests. DO NOT CHANGE THE NUMERIC VALUES OF THESE ENTRIES, NOR DELETE ANY -- add new ones at the end of the most relevant group. */ typedef enum { VG_USERREQ__RUNNING_ON_VALGRIND = 0x1001, VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002, /* These allow any function to be called from the simulated CPU but run on the real CPU. Nb: the first arg passed to the function is always the ThreadId of the running thread! So CLIENT_CALL0 actually requires a 1 arg function, etc. */ VG_USERREQ__CLIENT_CALL0 = 0x1101, VG_USERREQ__CLIENT_CALL1 = 0x1102, VG_USERREQ__CLIENT_CALL2 = 0x1103, VG_USERREQ__CLIENT_CALL3 = 0x1104, /* Can be useful in regression testing suites -- eg. can send Valgrind's output to /dev/null and still count errors. */ VG_USERREQ__COUNT_ERRORS = 0x1201, /* Allows the client program and/or gdbserver to execute a monitor command. */ VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202, /* Allows the client program to change a dynamic command line option. */ VG_USERREQ__CLO_CHANGE = 0x1203, /* These are useful and can be interpreted by any tool that tracks malloc() et al, by using vg_replace_malloc.c. */ VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301, VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b, VG_USERREQ__FREELIKE_BLOCK = 0x1302, /* Memory pool support. */ VG_USERREQ__CREATE_MEMPOOL = 0x1303, VG_USERREQ__DESTROY_MEMPOOL = 0x1304, VG_USERREQ__MEMPOOL_ALLOC = 0x1305, VG_USERREQ__MEMPOOL_FREE = 0x1306, VG_USERREQ__MEMPOOL_TRIM = 0x1307, VG_USERREQ__MOVE_MEMPOOL = 0x1308, VG_USERREQ__MEMPOOL_CHANGE = 0x1309, VG_USERREQ__MEMPOOL_EXISTS = 0x130a, /* Allow printfs to valgrind log. */ /* The first two pass the va_list argument by value, which assumes it is the same size as or smaller than a UWord, which generally isn't the case. Hence are deprecated. The second two pass the vargs by reference and so are immune to this problem. */ /* both :: char* fmt, va_list vargs (DEPRECATED) */ VG_USERREQ__PRINTF = 0x1401, VG_USERREQ__PRINTF_BACKTRACE = 0x1402, /* both :: char* fmt, va_list* vargs */ VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404, /* Stack support. */ VG_USERREQ__STACK_REGISTER = 0x1501, VG_USERREQ__STACK_DEREGISTER = 0x1502, VG_USERREQ__STACK_CHANGE = 0x1503, /* Wine support */ VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601, /* Querying of debug info. */ VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701, /* Disable/enable error reporting level. Takes a single Word arg which is the delta to this thread's error disablement indicator. Hence 1 disables or further disables errors, and -1 moves back towards enablement. Other values are not allowed. */ VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801, /* Some requests used for Valgrind internal, such as self-test or self-hosting. */ /* Initialise IR injection */ VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901, /* Used by Inner Valgrind to inform Outer Valgrind where to find the list of inner guest threads */ VG_USERREQ__INNER_THREADS = 0x1902 } Vg_ClientRequest; #if !defined(__GNUC__) # define __extension__ /* */ #endif /* Returns the number of Valgrinds this code is running under. That is, 0 if running natively, 1 if running under Valgrind, 2 if running under Valgrind which is running under another Valgrind, etc. */ #define RUNNING_ON_VALGRIND \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */, \ VG_USERREQ__RUNNING_ON_VALGRIND, \ 0, 0, 0, 0, 0) \ /* Discard translation of code in the range [_qzz_addr .. _qzz_addr + _qzz_len - 1]. Useful if you are debugging a JITter or some such, since it provides a way to make sure valgrind will retranslate the invalidated area. Returns no value. */ #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS, \ _qzz_addr, _qzz_len, 0, 0, 0) #define VALGRIND_INNER_THREADS(_qzz_addr) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__INNER_THREADS, \ _qzz_addr, 0, 0, 0, 0) /* These requests are for getting Valgrind itself to print something. Possibly with a backtrace. This is a really ugly hack. The return value is the number of characters printed, excluding the "**** " part at the start and the backtrace (if present). */ #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER) /* Modern GCC will optimize the static routine out if unused, and unused attribute will shut down warnings about it. */ static int VALGRIND_PRINTF(const char *format, ...) __attribute__((format(__printf__, 1, 2), __unused__)); #endif static int #if defined(_MSC_VER) __inline #endif VALGRIND_PRINTF(const char *format, ...) { #if defined(NVALGRIND) (void)format; return 0; #else /* NVALGRIND */ #if defined(_MSC_VER) || defined(__MINGW64__) uintptr_t _qzz_res; #else unsigned long _qzz_res; #endif va_list vargs; va_start(vargs, format); #if defined(_MSC_VER) || defined(__MINGW64__) _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_VALIST_BY_REF, (uintptr_t)format, (uintptr_t)&vargs, 0, 0, 0); #else _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_VALIST_BY_REF, (unsigned long)format, (unsigned long)&vargs, 0, 0, 0); #endif va_end(vargs); return (int)_qzz_res; #endif /* NVALGRIND */ } #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER) static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...) __attribute__((format(__printf__, 1, 2), __unused__)); #endif static int #if defined(_MSC_VER) __inline #endif VALGRIND_PRINTF_BACKTRACE(const char *format, ...) { #if defined(NVALGRIND) (void)format; return 0; #else /* NVALGRIND */ #if defined(_MSC_VER) || defined(__MINGW64__) uintptr_t _qzz_res; #else unsigned long _qzz_res; #endif va_list vargs; va_start(vargs, format); #if defined(_MSC_VER) || defined(__MINGW64__) _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF, (uintptr_t)format, (uintptr_t)&vargs, 0, 0, 0); #else _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF, (unsigned long)format, (unsigned long)&vargs, 0, 0, 0); #endif va_end(vargs); return (int)_qzz_res; #endif /* NVALGRIND */ } /* These requests allow control to move from the simulated CPU to the real CPU, calling an arbitrary function. Note that the current ThreadId is inserted as the first argument. So this call: VALGRIND_NON_SIMD_CALL2(f, arg1, arg2) requires f to have this signature: Word f(Word tid, Word arg1, Word arg2) where "Word" is a word-sized type. Note that these client requests are not entirely reliable. For example, if you call a function with them that subsequently calls printf(), there's a high chance Valgrind will crash. Generally, your prospects of these working are made higher if the called function does not refer to any global variables, and does not refer to any libc or other functions (printf et al). Any kind of entanglement with libc or dynamic linking is likely to have a bad outcome, for tricky reasons which we've grappled with a lot in the past. */ #define VALGRIND_NON_SIMD_CALL0(_qyy_fn) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL0, \ _qyy_fn, \ 0, 0, 0, 0) #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL1, \ _qyy_fn, \ _qyy_arg1, 0, 0, 0) #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL2, \ _qyy_fn, \ _qyy_arg1, _qyy_arg2, 0, 0) #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL3, \ _qyy_fn, \ _qyy_arg1, _qyy_arg2, \ _qyy_arg3, 0) /* Counts the number of errors that have been recorded by a tool. Nb: the tool must record the errors with VG_(maybe_record_error)() or VG_(unique_error)() for them to be counted. */ #define VALGRIND_COUNT_ERRORS \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR( \ 0 /* default return */, \ VG_USERREQ__COUNT_ERRORS, \ 0, 0, 0, 0, 0) /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing when heap blocks are allocated in order to give accurate results. This happens automatically for the standard allocator functions such as malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete, delete[], etc. But if your program uses a custom allocator, this doesn't automatically happen, and Valgrind will not do as well. For example, if you allocate superblocks with mmap() and then allocates chunks of the superblocks, all Valgrind's observations will be at the mmap() level and it won't know that the chunks should be considered separate entities. In Memcheck's case, that means you probably won't get heap block overrun detection (because there won't be redzones marked as unaddressable) and you definitely won't get any leak detection. The following client requests allow a custom allocator to be annotated so that it can be handled accurately by Valgrind. VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated by a malloc()-like function. For Memcheck (an illustrative case), this does two things: - It records that the block has been allocated. This means any addresses within the block mentioned in error messages will be identified as belonging to the block. It also means that if the block isn't freed it will be detected by the leak checker. - It marks the block as being addressable and undefined (if 'is_zeroed' is not set), or addressable and defined (if 'is_zeroed' is set). This controls how accesses to the block by the program are handled. 'addr' is the start of the usable block (ie. after any redzone), 'sizeB' is its size. 'rzB' is the redzone size if the allocator can apply redzones -- these are blocks of padding at the start and end of each block. Adding redzones is recommended as it makes it much more likely Valgrind will spot block overruns. `is_zeroed' indicates if the memory is zeroed (or filled with another predictable value), as is the case for calloc(). VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a heap block -- that will be used by the client program -- is allocated. It's best to put it at the outermost level of the allocator if possible; for example, if you have a function my_alloc() which calls internal_alloc(), and the client request is put inside internal_alloc(), stack traces relating to the heap block will contain entries for both my_alloc() and internal_alloc(), which is probably not what you want. For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out custom blocks from within a heap block, B, that has been allocated with malloc/calloc/new/etc, then block B will be *ignored* during leak-checking -- the custom blocks will take precedence. VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK. For Memcheck, it does two things: - It records that the block has been deallocated. This assumes that the block was annotated as having been allocated via VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued. - It marks the block as being unaddressable. VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a heap block is deallocated. VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For Memcheck, it does four things: - It records that the size of a block has been changed. This assumes that the block was annotated as having been allocated via VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued. - If the block shrunk, it marks the freed memory as being unaddressable. - If the block grew, it marks the new area as undefined and defines a red zone past the end of the new block. - The V-bits of the overlap between the old and the new block are preserved. VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block and before deallocation of the old block. In many cases, these three client requests will not be enough to get your allocator working well with Memcheck. More specifically, if your allocator writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call will be necessary to mark the memory as addressable just before the zeroing occurs, otherwise you'll get a lot of invalid write errors. For example, you'll need to do this if your allocator recycles freed blocks, but it zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK). Alternatively, if your allocator reuses freed blocks for allocator-internal data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary. Really, what's happening is a blurring of the lines between the client program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the memory should be considered unaddressable to the client program, but the allocator knows more than the rest of the client program and so may be able to safely access it. Extra client requests are necessary for Valgrind to understand the distinction between the allocator and the rest of the program. Ignored if addr == 0. */ #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK, \ addr, sizeB, rzB, is_zeroed, 0) /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details. Ignored if addr == 0. */ #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK, \ addr, oldSizeB, newSizeB, rzB, 0) /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details. Ignored if addr == 0. */ #define VALGRIND_FREELIKE_BLOCK(addr, rzB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK, \ addr, rzB, 0, 0, 0) /* Create a memory pool. */ #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL, \ pool, rzB, is_zeroed, 0, 0) /* Create a memory pool with some flags specifying extended behaviour. When flags is zero, the behaviour is identical to VALGRIND_CREATE_MEMPOOL. The flag VALGRIND_MEMPOOL_METAPOOL specifies that the pieces of memory associated with the pool using VALGRIND_MEMPOOL_ALLOC will be used by the application as superblocks to dole out MALLOC_LIKE blocks using VALGRIND_MALLOCLIKE_BLOCK. In other words, a meta pool is a "2 levels" pool : first level is the blocks described by VALGRIND_MEMPOOL_ALLOC. The second level blocks are described using VALGRIND_MALLOCLIKE_BLOCK. Note that the association between the pool and the second level blocks is implicit : second level blocks will be located inside first level blocks. It is necessary to use the VALGRIND_MEMPOOL_METAPOOL flag for such 2 levels pools, as otherwise valgrind will detect overlapping memory blocks, and will abort execution (e.g. during leak search). Such a meta pool can also be marked as an 'auto free' pool using the flag VALGRIND_MEMPOOL_AUTO_FREE, which must be OR-ed together with the VALGRIND_MEMPOOL_METAPOOL. For an 'auto free' pool, VALGRIND_MEMPOOL_FREE will automatically free the second level blocks that are contained inside the first level block freed with VALGRIND_MEMPOOL_FREE. In other words, calling VALGRIND_MEMPOOL_FREE will cause implicit calls to VALGRIND_FREELIKE_BLOCK for all the second level blocks included in the first level block. Note: it is an error to use the VALGRIND_MEMPOOL_AUTO_FREE flag without the VALGRIND_MEMPOOL_METAPOOL flag. */ #define VALGRIND_MEMPOOL_AUTO_FREE 1 #define VALGRIND_MEMPOOL_METAPOOL 2 #define VALGRIND_CREATE_MEMPOOL_EXT(pool, rzB, is_zeroed, flags) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL, \ pool, rzB, is_zeroed, flags, 0) /* Destroy a memory pool. */ #define VALGRIND_DESTROY_MEMPOOL(pool) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL, \ pool, 0, 0, 0, 0) /* Associate a piece of memory with a memory pool. */ #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC, \ pool, addr, size, 0, 0) /* Disassociate a piece of memory from a memory pool. */ #define VALGRIND_MEMPOOL_FREE(pool, addr) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE, \ pool, addr, 0, 0, 0) /* Disassociate any pieces outside a particular range. */ #define VALGRIND_MEMPOOL_TRIM(pool, addr, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM, \ pool, addr, size, 0, 0) /* Resize and/or move a piece associated with a memory pool. */ #define VALGRIND_MOVE_MEMPOOL(poolA, poolB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL, \ poolA, poolB, 0, 0, 0) /* Resize and/or move a piece associated with a memory pool. */ #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE, \ pool, addrA, addrB, size, 0) /* Return 1 if a mempool exists, else 0. */ #define VALGRIND_MEMPOOL_EXISTS(pool) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__MEMPOOL_EXISTS, \ pool, 0, 0, 0, 0) /* Mark a piece of memory as being a stack. Returns a stack id. start is the lowest addressable stack byte, end is the highest addressable stack byte. */ #define VALGRIND_STACK_REGISTER(start, end) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__STACK_REGISTER, \ start, end, 0, 0, 0) /* Unmark the piece of memory associated with a stack id as being a stack. */ #define VALGRIND_STACK_DEREGISTER(id) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \ id, 0, 0, 0, 0) /* Change the start and end address of the stack id. start is the new lowest addressable stack byte, end is the new highest addressable stack byte. */ #define VALGRIND_STACK_CHANGE(id, start, end) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE, \ id, start, end, 0, 0) /* Load PDB debug info for Wine PE image_map. */ #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \ fd, ptr, total_size, delta, 0) /* Map a code address to a source file name and line number. buf64 must point to a 64-byte buffer in the caller's address space. The result will be dumped in there and is guaranteed to be zero terminated. If no info is found, the first byte is set to zero. */ #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__MAP_IP_TO_SRCLOC, \ addr, buf64, 0, 0, 0) /* Disable error reporting for this thread. Behaves in a stack like way, so you can safely call this multiple times provided that VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times to re-enable reporting. The first call of this macro disables reporting. Subsequent calls have no effect except to increase the number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable reporting. Child threads do not inherit this setting from their parents -- they are always created with reporting enabled. */ #define VALGRIND_DISABLE_ERROR_REPORTING \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \ 1, 0, 0, 0, 0) /* Re-enable error reporting, as per comments on VALGRIND_DISABLE_ERROR_REPORTING. */ #define VALGRIND_ENABLE_ERROR_REPORTING \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \ -1, 0, 0, 0, 0) /* Execute a monitor command from the client program. If a connection is opened with GDB, the output will be sent according to the output mode set for vgdb. If no connection is opened, output will go to the log output. Returns 1 if command not recognised, 0 otherwise. */ #define VALGRIND_MONITOR_COMMAND(command) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \ command, 0, 0, 0, 0) /* Change the value of a dynamic command line option. Note that unknown or not dynamically changeable options will cause a warning message to be output. */ #define VALGRIND_CLO_CHANGE(option) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CLO_CHANGE, \ option, 0, 0, 0, 0) #undef PLAT_x86_darwin #undef PLAT_amd64_darwin #undef PLAT_x86_win32 #undef PLAT_amd64_win64 #undef PLAT_x86_linux #undef PLAT_amd64_linux #undef PLAT_ppc32_linux #undef PLAT_ppc64be_linux #undef PLAT_ppc64le_linux #undef PLAT_arm_linux #undef PLAT_s390x_linux #undef PLAT_mips32_linux #undef PLAT_mips64_linux #undef PLAT_nanomips_linux #undef PLAT_x86_solaris #undef PLAT_amd64_solaris #endif /* __VALGRIND_H */ codspeed-4.4.1/instrument-hooks/example/instrument-hooks/includes/zig.h000064400000000000000000004662131046102023000245210ustar 00000000000000#undef linux #include #include #if defined(_MSC_VER) #define zig_msvc #elif defined(__clang__) #define zig_clang #define zig_gnuc #elif defined(__GNUC__) #define zig_gcc #define zig_gnuc #elif defined(__IBMC__) #define zig_xlc #elif defined(__TINYC__) #define zig_tinyc #elif defined(__slimcc__) #define zig_slimcc #endif #if defined(__aarch64__) || (defined(zig_msvc) && defined(_M_ARM64)) #define zig_aarch64 #elif defined(__thumb__) || (defined(zig_msvc) && defined(_M_ARM)) #define zig_thumb #define zig_arm #elif defined(__arm__) #define zig_arm #elif defined(__hexagon__) #define zig_hexagon #elif defined(__loongarch32) #define zig_loongarch32 #define zig_loongarch #elif defined(__loongarch64) #define zig_loongarch64 #define zig_loongarch #elif defined(__mips64) #define zig_mips64 #define zig_mips #elif defined(__mips__) #define zig_mips32 #define zig_mips #elif defined(__powerpc64__) #define zig_powerpc64 #define zig_powerpc #elif defined(__powerpc__) #define zig_powerpc32 #define zig_powerpc #elif defined(__riscv) && __riscv_xlen == 32 #define zig_riscv32 #define zig_riscv #elif defined(__riscv) && __riscv_xlen == 64 #define zig_riscv64 #define zig_riscv #elif defined(__s390x__) #define zig_s390x #elif defined(__sparc__) && defined(__arch64__) #define zig_sparc64 #define zig_sparc #elif defined(__sparc__) #define zig_sparc32 #define zig_sparc #elif defined(__wasm32__) #define zig_wasm32 #define zig_wasm #elif defined(__wasm64__) #define zig_wasm64 #define zig_wasm #elif defined(__i386__) || (defined(zig_msvc) && defined(_M_IX86)) #define zig_x86_32 #define zig_x86 #elif defined (__x86_64__) || (defined(zig_msvc) && defined(_M_X64)) #define zig_x86_64 #define zig_x86 #endif #if defined(zig_msvc) || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define zig_little_endian 1 #define zig_big_endian 0 #else #define zig_little_endian 0 #define zig_big_endian 1 #endif #if defined(_AIX) #define zig_aix #elif defined(__MACH__) #define zig_darwin #elif defined(__DragonFly__) #define zig_dragonfly #define zig_bsd #elif defined(__EMSCRIPTEN__) #define zig_emscripten #elif defined(__FreeBSD__) #define zig_freebsd #define zig_bsd #elif defined(__Fuchsia__) #define zig_fuchsia #elif defined(__HAIKU__) #define zig_haiku #elif defined(__gnu_hurd__) #define zig_hurd #elif defined(__linux__) #define zig_linux #elif defined(__NetBSD__) #define zig_netbsd #define zig_bsd #elif defined(__OpenBSD__) #define zig_openbsd #define zig_bsd #elif defined(__SVR4) #define zig_solaris #elif defined(__wasi__) #define zig_wasi #elif defined(_WIN32) #define zig_windows #elif defined(__MVS__) #define zig_zos #endif #if defined(zig_windows) #define zig_coff #elif defined(__ELF__) #define zig_elf #elif defined(zig_zos) #define zig_goff #elif defined(zig_darwin) #define zig_macho #elif defined(zig_aix) #define zig_xcoff #endif #define zig_concat(lhs, rhs) lhs##rhs #define zig_expand_concat(lhs, rhs) zig_concat(lhs, rhs) #if defined(__has_include) #define zig_has_include(include) __has_include(include) #else #define zig_has_include(include) 0 #endif #if defined(__has_builtin) #define zig_has_builtin(builtin) __has_builtin(__builtin_##builtin) #else #define zig_has_builtin(builtin) 0 #endif #define zig_expand_has_builtin(b) zig_has_builtin(b) #if defined(__has_attribute) #define zig_has_attribute(attribute) __has_attribute(attribute) #else #define zig_has_attribute(attribute) 0 #endif #if __STDC_VERSION__ >= 202311L #define zig_threadlocal thread_local #elif __STDC_VERSION__ >= 201112L #define zig_threadlocal _Thread_local #elif defined(zig_gnuc) || defined(zig_slimcc) #define zig_threadlocal __thread #elif defined(zig_msvc) #define zig_threadlocal __declspec(thread) #else #define zig_threadlocal zig_threadlocal_unavailable #endif #if defined(zig_msvc) #define zig_const_arr #define zig_callconv(c) __##c #else #define zig_const_arr static const #define zig_callconv(c) __attribute__((c)) #endif #if zig_has_attribute(naked) || defined(zig_gcc) #define zig_naked_decl __attribute__((naked)) #define zig_naked __attribute__((naked)) #elif defined(zig_msvc) #define zig_naked_decl #define zig_naked __declspec(naked) #else #define zig_naked_decl zig_naked_unavailable #define zig_naked zig_naked_unavailable #endif #if zig_has_attribute(cold) #define zig_cold __attribute__((cold)) #else #define zig_cold #endif #if zig_has_attribute(flatten) #define zig_maybe_flatten __attribute__((flatten)) #else #define zig_maybe_flatten #endif #if zig_has_attribute(noinline) #define zig_never_inline __attribute__((noinline)) zig_maybe_flatten #elif defined(zig_msvc) #define zig_never_inline __declspec(noinline) zig_maybe_flatten #else #define zig_never_inline zig_never_inline_unavailable #endif #if zig_has_attribute(not_tail_called) #define zig_never_tail __attribute__((not_tail_called)) zig_never_inline #else #define zig_never_tail zig_never_tail_unavailable #endif #if zig_has_attribute(musttail) #define zig_always_tail __attribute__((musttail)) #else #define zig_always_tail zig_always_tail_unavailable #endif #if __STDC_VERSION__ >= 199901L #define zig_restrict restrict #elif defined(zig_gnuc) || defined(zig_tinyc) #define zig_restrict __restrict #else #define zig_restrict #endif #if zig_has_attribute(no_builtin) #define zig_no_builtin __attribute__((no_builtin)) #else #define zig_no_builtin #endif #if zig_has_attribute(aligned) || defined(zig_tinyc) #define zig_under_align(alignment) __attribute__((aligned(alignment))) #elif defined(zig_msvc) #define zig_under_align(alignment) __declspec(align(alignment)) #else #define zig_under_align zig_align_unavailable #endif #if __STDC_VERSION__ >= 202311L #define zig_align(alignment) alignas(alignment) #elif __STDC_VERSION__ >= 201112L #define zig_align(alignment) _Alignas(alignment) #else #define zig_align(alignment) zig_under_align(alignment) #endif #if zig_has_attribute(aligned) || defined(zig_tinyc) #define zig_align_fn(alignment) __attribute__((aligned(alignment))) #elif defined(zig_msvc) #define zig_align_fn(alignment) #else #define zig_align_fn zig_align_fn_unavailable #endif #if zig_has_attribute(packed) || defined(zig_tinyc) #define zig_packed(definition) __attribute__((packed)) definition #elif defined(zig_msvc) #define zig_packed(definition) __pragma(pack(1)) definition __pragma(pack()) #else #define zig_packed(definition) zig_packed_unavailable #endif #if zig_has_attribute(section) || defined(zig_tinyc) #define zig_linksection(name) __attribute__((section(name))) #define zig_linksection_fn zig_linksection #elif defined(zig_msvc) #define zig_linksection(name) __pragma(section(name, read, write)) __declspec(allocate(name)) #define zig_linksection_fn(name) __pragma(section(name, read, execute)) __declspec(code_seg(name)) #else #define zig_linksection(name) zig_linksection_unavailable #define zig_linksection_fn zig_linksection #endif #if zig_has_builtin(unreachable) || defined(zig_gcc) || defined(zig_tinyc) #define zig_unreachable() __builtin_unreachable() #elif defined(zig_msvc) #define zig_unreachable() __assume(0) #else #define zig_unreachable() #endif #if defined(__cplusplus) #define zig_extern extern "C" #else #define zig_extern extern #endif #if defined(zig_msvc) #if defined(zig_x86_64) #define zig_mangle_c(symbol) symbol #else /* zig_x86_64 */ #define zig_mangle_c(symbol) "_" symbol #endif /* zig_x86_64 */ #else /* zig_msvc */ #if defined(zig_macho) #define zig_mangle_c(symbol) "_" symbol #else /* zig_macho */ #define zig_mangle_c(symbol) symbol #endif /* zig_macho */ #endif /* zig_msvc */ #if defined(zig_msvc) #define zig_export(symbol, name) ; \ __pragma(comment(linker, "/alternatename:" zig_mangle_c(name) "=" zig_mangle_c(symbol))) #elif (zig_has_attribute(alias) || defined(zig_tinyc)) && !defined(zig_macho) #define zig_export(symbol, name) __attribute__((alias(symbol))) #else #define zig_export(symbol, name) ; \ __asm(zig_mangle_c(name) " = " zig_mangle_c(symbol)) #endif #define zig_mangled_tentative zig_mangled #define zig_mangled_final zig_mangled #if defined(zig_msvc) #define zig_mangled(mangled, unmangled) ; \ zig_export(#mangled, unmangled) #define zig_mangled_export(mangled, unmangled, symbol) \ zig_export(unmangled, #mangled) \ zig_export(symbol, unmangled) #else /* zig_msvc */ #define zig_mangled(mangled, unmangled) __asm(zig_mangle_c(unmangled)) #define zig_mangled_export(mangled, unmangled, symbol) \ zig_mangled_final(mangled, unmangled) \ zig_export(symbol, unmangled) #endif /* zig_msvc */ #if defined(zig_msvc) #define zig_import(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type fn_name sig_args;\ __pragma(comment(linker, "/alternatename:" zig_mangle_c(#fn_name) "=" zig_mangle_c(#libc_name))); #define zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) zig_import(Type, fn_name, libc_name, sig_args, call_args) #else /* zig_msvc */ #define zig_import(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type fn_name sig_args __asm(zig_mangle_c(#libc_name)); #define zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type libc_name sig_args; \ static inline Type fn_name sig_args { return libc_name call_args; } #endif #define zig_expand_import_0(Type, fn_name, libc_name, sig_args, call_args) zig_import(Type, fn_name, libc_name, sig_args, call_args) #define zig_expand_import_1(Type, fn_name, libc_name, sig_args, call_args) zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) #if zig_has_attribute(weak) || defined(zig_gcc) || defined(zig_tinyc) #define zig_weak_linkage __attribute__((weak)) #define zig_weak_linkage_fn __attribute__((weak)) #elif defined(zig_msvc) #define zig_weak_linkage __declspec(selectany) #define zig_weak_linkage_fn #else #define zig_weak_linkage zig_weak_linkage_unavailable #define zig_weak_linkage_fn zig_weak_linkage_unavailable #endif #if defined(zig_gnuc) || defined(zig_tinyc) || defined(zig_slimcc) #define zig_gnuc_asm #endif #if zig_has_builtin(trap) #define zig_trap() __builtin_trap() #elif defined(zig_msvc) #if defined(zig_x86) #define zig_trap() __ud2() #else #define zig_trap() __fastfail(7) #endif #elif defined(zig_gnuc_asm) #if defined(zig_thumb) #define zig_trap() __asm__ volatile("udf #0xfe") #elif defined(zig_arm) || defined(zig_aarch64) #define zig_trap() __asm__ volatile("udf #0xfdee") #elif defined(zig_hexagon) #define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)") #elif defined(zig_loongarch) || defined(zig_powerpc) #define zig_trap() __asm__ volatile(".word 0x0") #elif defined(zig_mips) #define zig_trap() __asm__ volatile(".word 0x3d") #elif defined(zig_riscv) #define zig_trap() __asm__ volatile("unimp") #elif defined(zig_s390x) #define zig_trap() __asm__ volatile("j 0x2") #elif defined(zig_sparc) #define zig_trap() __asm__ volatile("illtrap") #elif defined(zig_x86) #define zig_trap() __asm__ volatile("ud2") #else #define zig_trap() zig_trap_unavailable #endif #else #define zig_trap() zig_trap_unavailable #endif #if zig_has_builtin(debugtrap) #define zig_breakpoint() __builtin_debugtrap() #elif defined(zig_msvc) #define zig_breakpoint() __debugbreak() #elif defined(zig_gnuc_asm) #if defined(zig_arm) #define zig_breakpoint() __asm__ volatile("bkpt #0x0") #elif defined(zig_aarch64) #define zig_breakpoint() __asm__ volatile("brk #0xf000") #elif defined(zig_hexagon) #define zig_breakpoint() __asm__ volatile("brkpt") #elif defined(zig_loongarch) #define zig_breakpoint() __asm__ volatile("break 0x0") #elif defined(zig_mips) #define zig_breakpoint() __asm__ volatile("break") #elif defined(zig_powerpc) #define zig_breakpoint() __asm__ volatile("trap") #elif defined(zig_riscv) #define zig_breakpoint() __asm__ volatile("ebreak") #elif defined(zig_s390x) #define zig_breakpoint() __asm__ volatile("j 0x6") #elif defined(zig_sparc) #define zig_breakpoint() __asm__ volatile("ta 0x1") #elif defined(zig_x86) #define zig_breakpoint() __asm__ volatile("int $0x3") #else #define zig_breakpoint() zig_breakpoint_unavailable #endif #else #define zig_breakpoint() zig_breakpoint_unavailable #endif #if zig_has_builtin(return_address) || defined(zig_gcc) || defined(zig_tinyc) #define zig_return_address() __builtin_extract_return_addr(__builtin_return_address(0)) #elif defined(zig_msvc) #define zig_return_address() _ReturnAddress() #else #define zig_return_address() 0 #endif #if zig_has_builtin(frame_address) || defined(zig_gcc) || defined(zig_tinyc) #define zig_frame_address() __builtin_frame_address(0) #elif defined(zig_msvc) #define zig_frame_address() _AddressOfReturnAddress() #else #define zig_frame_address() 0 #endif #if zig_has_builtin(prefetch) || defined(zig_gcc) #define zig_prefetch(addr, rw, locality) __builtin_prefetch(addr, rw, locality) #else #define zig_prefetch(addr, rw, locality) #endif #if zig_has_builtin(memory_size) && zig_has_builtin(memory_grow) #define zig_wasm_memory_size(index) __builtin_wasm_memory_size(index) #define zig_wasm_memory_grow(index, delta) __builtin_wasm_memory_grow(index, delta) #else #define zig_wasm_memory_size(index) zig_unimplemented() #define zig_wasm_memory_grow(index, delta) zig_unimplemented() #endif #if __STDC_VERSION__ >= 202311L #define zig_noreturn [[noreturn]] #elif __STDC_VERSION__ >= 201112L #define zig_noreturn _Noreturn #elif zig_has_attribute(noreturn) || defined(zig_gcc) || defined(zig_tinyc) #define zig_noreturn __attribute__((noreturn)) #elif defined(zig_msvc) #define zig_noreturn __declspec(noreturn) #else #define zig_noreturn #endif #define zig_compiler_rt_abbrev_uint32_t si #define zig_compiler_rt_abbrev_int32_t si #define zig_compiler_rt_abbrev_uint64_t di #define zig_compiler_rt_abbrev_int64_t di #define zig_compiler_rt_abbrev_zig_u128 ti #define zig_compiler_rt_abbrev_zig_i128 ti #define zig_compiler_rt_abbrev_zig_f16 hf #define zig_compiler_rt_abbrev_zig_f32 sf #define zig_compiler_rt_abbrev_zig_f64 df #define zig_compiler_rt_abbrev_zig_f80 xf #define zig_compiler_rt_abbrev_zig_f128 tf zig_extern void *memcpy (void *zig_restrict, void const *zig_restrict, size_t); zig_extern void *memset (void *, int, size_t); /* ================ Bool and 8/16/32/64-bit Integer Support ================= */ #include #define zig_bitSizeOf(T) (CHAR_BIT * sizeof(T)) #if __STDC_VERSION__ >= 202311L /* bool, true, and false are provided by the language. */ #elif __STDC_VERSION__ >= 199901L || zig_has_include() #include #else typedef char bool; #define false 0 #define true 1 #endif #if __STDC_VERSION__ >= 199901L || defined(zig_msvc) || zig_has_include() #include #else #if SCHAR_MIN == ~0x7F && SCHAR_MAX == 0x7F && UCHAR_MAX == 0xFF typedef unsigned char uint8_t; typedef signed char int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif SHRT_MIN == ~0x7F && SHRT_MAX == 0x7F && USHRT_MAX == 0xFF typedef unsigned short uint8_t; typedef signed short int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif INT_MIN == ~0x7F && INT_MAX == 0x7F && UINT_MAX == 0xFF typedef unsigned int uint8_t; typedef signed int int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif LONG_MIN == ~0x7F && LONG_MAX == 0x7F && ULONG_MAX == 0xFF typedef unsigned long uint8_t; typedef signed long int8_t; #define INT8_C(c) c##L #define UINT8_C(c) c##LU #elif LLONG_MIN == ~0x7F && LLONG_MAX == 0x7F && ULLONG_MAX == 0xFF typedef unsigned long long uint8_t; typedef signed long long int8_t; #define INT8_C(c) c##LL #define UINT8_C(c) c##LLU #endif #define INT8_MIN (~INT8_C(0x7F)) #define INT8_MAX ( INT8_C(0x7F)) #define UINT8_MAX ( INT8_C(0xFF)) #if SCHAR_MIN == ~0x7FFF && SCHAR_MAX == 0x7FFF && UCHAR_MAX == 0xFFFF typedef unsigned char uint16_t; typedef signed char int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif SHRT_MIN == ~0x7FFF && SHRT_MAX == 0x7FFF && USHRT_MAX == 0xFFFF typedef unsigned short uint16_t; typedef signed short int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif INT_MIN == ~0x7FFF && INT_MAX == 0x7FFF && UINT_MAX == 0xFFFF typedef unsigned int uint16_t; typedef signed int int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif LONG_MIN == ~0x7FFF && LONG_MAX == 0x7FFF && ULONG_MAX == 0xFFFF typedef unsigned long uint16_t; typedef signed long int16_t; #define INT16_C(c) c##L #define UINT16_C(c) c##LU #elif LLONG_MIN == ~0x7FFF && LLONG_MAX == 0x7FFF && ULLONG_MAX == 0xFFFF typedef unsigned long long uint16_t; typedef signed long long int16_t; #define INT16_C(c) c##LL #define UINT16_C(c) c##LLU #endif #define INT16_MIN (~INT16_C(0x7FFF)) #define INT16_MAX ( INT16_C(0x7FFF)) #define UINT16_MAX ( INT16_C(0xFFFF)) #if SCHAR_MIN == ~0x7FFFFFFF && SCHAR_MAX == 0x7FFFFFFF && UCHAR_MAX == 0xFFFFFFFF typedef unsigned char uint32_t; typedef signed char int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif SHRT_MIN == ~0x7FFFFFFF && SHRT_MAX == 0x7FFFFFFF && USHRT_MAX == 0xFFFFFFFF typedef unsigned short uint32_t; typedef signed short int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif INT_MIN == ~0x7FFFFFFF && INT_MAX == 0x7FFFFFFF && UINT_MAX == 0xFFFFFFFF typedef unsigned int uint32_t; typedef signed int int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif LONG_MIN == ~0x7FFFFFFF && LONG_MAX == 0x7FFFFFFF && ULONG_MAX == 0xFFFFFFFF typedef unsigned long uint32_t; typedef signed long int32_t; #define INT32_C(c) c##L #define UINT32_C(c) c##LU #elif LLONG_MIN == ~0x7FFFFFFF && LLONG_MAX == 0x7FFFFFFF && ULLONG_MAX == 0xFFFFFFFF typedef unsigned long long uint32_t; typedef signed long long int32_t; #define INT32_C(c) c##LL #define UINT32_C(c) c##LLU #endif #define INT32_MIN (~INT32_C(0x7FFFFFFF)) #define INT32_MAX ( INT32_C(0x7FFFFFFF)) #define UINT32_MAX ( INT32_C(0xFFFFFFFF)) #if SCHAR_MIN == ~0x7FFFFFFFFFFFFFFF && SCHAR_MAX == 0x7FFFFFFFFFFFFFFF && UCHAR_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned char uint64_t; typedef signed char int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif SHRT_MIN == ~0x7FFFFFFFFFFFFFFF && SHRT_MAX == 0x7FFFFFFFFFFFFFFF && USHRT_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned short uint64_t; typedef signed short int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif INT_MIN == ~0x7FFFFFFFFFFFFFFF && INT_MAX == 0x7FFFFFFFFFFFFFFF && UINT_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned int uint64_t; typedef signed int int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif LONG_MIN == ~0x7FFFFFFFFFFFFFFF && LONG_MAX == 0x7FFFFFFFFFFFFFFF && ULONG_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned long uint64_t; typedef signed long int64_t; #define INT64_C(c) c##L #define UINT64_C(c) c##LU #elif LLONG_MIN == ~0x7FFFFFFFFFFFFFFF && LLONG_MAX == 0x7FFFFFFFFFFFFFFF && ULLONG_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned long long uint64_t; typedef signed long long int64_t; #define INT64_C(c) c##LL #define UINT64_C(c) c##LLU #endif #define INT64_MIN (~INT64_C(0x7FFFFFFFFFFFFFFF)) #define INT64_MAX ( INT64_C(0x7FFFFFFFFFFFFFFF)) #define UINT64_MAX ( INT64_C(0xFFFFFFFFFFFFFFFF)) typedef size_t uintptr_t; typedef ptrdiff_t intptr_t; #endif #define zig_minInt_i8 INT8_MIN #define zig_maxInt_i8 INT8_MAX #define zig_minInt_u8 UINT8_C(0) #define zig_maxInt_u8 UINT8_MAX #define zig_minInt_i16 INT16_MIN #define zig_maxInt_i16 INT16_MAX #define zig_minInt_u16 UINT16_C(0) #define zig_maxInt_u16 UINT16_MAX #define zig_minInt_i32 INT32_MIN #define zig_maxInt_i32 INT32_MAX #define zig_minInt_u32 UINT32_C(0) #define zig_maxInt_u32 UINT32_MAX #define zig_minInt_i64 INT64_MIN #define zig_maxInt_i64 INT64_MAX #define zig_minInt_u64 UINT64_C(0) #define zig_maxInt_u64 UINT64_MAX #define zig_intLimit(s, w, limit, bits) zig_shr_##s##w(zig_##limit##Int_##s##w, w - (bits)) #define zig_minInt_i(w, bits) zig_intLimit(i, w, min, bits) #define zig_maxInt_i(w, bits) zig_intLimit(i, w, max, bits) #define zig_minInt_u(w, bits) zig_intLimit(u, w, min, bits) #define zig_maxInt_u(w, bits) zig_intLimit(u, w, max, bits) #define zig_operator(Type, RhsType, operation, operator) \ static inline Type zig_##operation(Type lhs, RhsType rhs) { \ return lhs operator rhs; \ } #define zig_basic_operator(Type, operation, operator) \ zig_operator(Type, Type, operation, operator) #define zig_shift_operator(Type, operation, operator) \ zig_operator(Type, uint8_t, operation, operator) #define zig_int_helpers(w, PromotedUnsigned) \ zig_basic_operator(uint##w##_t, and_u##w, &) \ zig_basic_operator( int##w##_t, and_i##w, &) \ zig_basic_operator(uint##w##_t, or_u##w, |) \ zig_basic_operator( int##w##_t, or_i##w, |) \ zig_basic_operator(uint##w##_t, xor_u##w, ^) \ zig_basic_operator( int##w##_t, xor_i##w, ^) \ zig_shift_operator(uint##w##_t, shl_u##w, <<) \ zig_shift_operator( int##w##_t, shl_i##w, <<) \ zig_shift_operator(uint##w##_t, shr_u##w, >>) \ \ static inline int##w##_t zig_shr_i##w(int##w##_t lhs, uint8_t rhs) { \ int##w##_t sign_mask = lhs < INT##w##_C(0) ? -INT##w##_C(1) : INT##w##_C(0); \ return ((lhs ^ sign_mask) >> rhs) ^ sign_mask; \ } \ \ static inline uint##w##_t zig_not_u##w(uint##w##_t val, uint8_t bits) { \ return val ^ zig_maxInt_u(w, bits); \ } \ \ static inline int##w##_t zig_not_i##w(int##w##_t val, uint8_t bits) { \ (void)bits; \ return ~val; \ } \ \ static inline uint##w##_t zig_wrap_u##w(uint##w##_t val, uint8_t bits) { \ return val & zig_maxInt_u(w, bits); \ } \ \ static inline int##w##_t zig_wrap_i##w(int##w##_t val, uint8_t bits) { \ return (val & UINT##w##_C(1) << (bits - UINT8_C(1))) != 0 \ ? val | zig_minInt_i(w, bits) : val & zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_abs_i##w(int##w##_t val) { \ return (val < 0) ? -(uint##w##_t)val : (uint##w##_t)val; \ } \ \ zig_basic_operator(uint##w##_t, div_floor_u##w, /) \ \ static inline int##w##_t zig_div_floor_i##w(int##w##_t lhs, int##w##_t rhs) { \ return lhs / rhs + (lhs % rhs != INT##w##_C(0) ? zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ } \ \ zig_basic_operator(uint##w##_t, mod_u##w, %) \ \ static inline int##w##_t zig_mod_i##w(int##w##_t lhs, int##w##_t rhs) { \ int##w##_t rem = lhs % rhs; \ return rem + (rem != INT##w##_C(0) ? rhs & zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ } \ \ static inline uint##w##_t zig_shlw_u##w(uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \ return zig_wrap_u##w(zig_shl_u##w(lhs, rhs), bits); \ } \ \ static inline int##w##_t zig_shlw_i##w(int##w##_t lhs, uint8_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)zig_shl_u##w((uint##w##_t)lhs, rhs), bits); \ } \ \ static inline uint##w##_t zig_addw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w(lhs + rhs, bits); \ } \ \ static inline int##w##_t zig_addw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs + (uint##w##_t)rhs), bits); \ } \ \ static inline uint##w##_t zig_subw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w(lhs - rhs, bits); \ } \ \ static inline int##w##_t zig_subw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs - (uint##w##_t)rhs), bits); \ } \ \ static inline uint##w##_t zig_mulw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w((PromotedUnsigned)lhs * rhs, bits); \ } \ \ static inline int##w##_t zig_mulw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs * (uint##w##_t)rhs), bits); \ } #if UINT8_MAX <= UINT_MAX zig_int_helpers(8, unsigned int) #elif UINT8_MAX <= ULONG_MAX zig_int_helpers(8, unsigned long) #elif UINT8_MAX <= ULLONG_MAX zig_int_helpers(8, unsigned long long) #else zig_int_helpers(8, uint8_t) #endif #if UINT16_MAX <= UINT_MAX zig_int_helpers(16, unsigned int) #elif UINT16_MAX <= ULONG_MAX zig_int_helpers(16, unsigned long) #elif UINT16_MAX <= ULLONG_MAX zig_int_helpers(16, unsigned long long) #else zig_int_helpers(16, uint16_t) #endif #if UINT32_MAX <= UINT_MAX zig_int_helpers(32, unsigned int) #elif UINT32_MAX <= ULONG_MAX zig_int_helpers(32, unsigned long) #elif UINT32_MAX <= ULLONG_MAX zig_int_helpers(32, unsigned long long) #else zig_int_helpers(32, uint32_t) #endif #if UINT64_MAX <= UINT_MAX zig_int_helpers(64, unsigned int) #elif UINT64_MAX <= ULONG_MAX zig_int_helpers(64, unsigned long) #elif UINT64_MAX <= ULLONG_MAX zig_int_helpers(64, unsigned long long) #else zig_int_helpers(64, uint64_t) #endif static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_addw_u32(lhs, rhs, bits); return *res < lhs; #endif } zig_extern int32_t __addosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __addosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_addw_u64(lhs, rhs, bits); return *res < lhs; #endif } zig_extern int64_t __addodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __addodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_addo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_addo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_addo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_addo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_addo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_addo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_addo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_addo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_subw_u32(lhs, rhs, bits); return *res > lhs; #endif } zig_extern int32_t __subosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __subosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_subw_u64(lhs, rhs, bits); return *res > lhs; #endif } zig_extern int64_t __subodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __subodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_subo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_subo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_subo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_subo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_subo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_subo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_subo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_subo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } static inline bool zig_mulo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_mulw_u32(lhs, rhs, bits); return rhs != UINT32_C(0) && lhs > zig_maxInt_u(32, bits) / rhs; #endif } zig_extern int32_t __mulosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_mulo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __mulosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_mulo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_mulw_u64(lhs, rhs, bits); return rhs != UINT64_C(0) && lhs > zig_maxInt_u(64, bits) / rhs; #endif } zig_extern int64_t __mulodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_mulo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __mulodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_mulo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_mulo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_mulo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_mulo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_mulo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_mulo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_mulo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } #define zig_int_builtins(w) \ static inline bool zig_shlo_u##w(uint##w##_t *res, uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \ *res = zig_shlw_u##w(lhs, rhs, bits); \ return lhs > zig_maxInt_u(w, bits) >> rhs; \ } \ \ static inline bool zig_shlo_i##w(int##w##_t *res, int##w##_t lhs, uint8_t rhs, uint8_t bits) { \ *res = zig_shlw_i##w(lhs, rhs, bits); \ int##w##_t mask = (int##w##_t)(UINT##w##_MAX << (bits - rhs - 1)); \ return (lhs & mask) != INT##w##_C(0) && (lhs & mask) != mask; \ } \ \ static inline uint##w##_t zig_shls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ if (rhs >= bits) return lhs != UINT##w##_C(0) ? zig_maxInt_u(w, bits) : lhs; \ return zig_shlo_u##w(&res, lhs, (uint8_t)rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_shls_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if ((uint##w##_t)rhs < (uint##w##_t)bits && !zig_shlo_i##w(&res, lhs, (uint8_t)rhs, bits)) return res; \ return lhs < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_adds_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_addo_u##w(&res, lhs, rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_adds_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_addo_i##w(&res, lhs, rhs, bits)) return res; \ return res >= INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_subs_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_subo_u##w(&res, lhs, rhs, bits) ? zig_minInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_subs_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_subo_i##w(&res, lhs, rhs, bits)) return res; \ return res >= INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_muls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_mulo_u##w(&res, lhs, rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_muls_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_mulo_i##w(&res, lhs, rhs, bits)) return res; \ return (lhs ^ rhs) < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } zig_int_builtins(8) zig_int_builtins(16) zig_int_builtins(32) zig_int_builtins(64) #define zig_builtin8(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin8; #define zig_builtin16(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin16; #if INT_MIN <= INT32_MIN #define zig_builtin32(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin32; #elif LONG_MIN <= INT32_MIN #define zig_builtin32(name, val) __builtin_##name##l(val) typedef unsigned long zig_Builtin32; #endif #if INT_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin64; #elif LONG_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name##l(val) typedef unsigned long zig_Builtin64; #elif LLONG_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name##ll(val) typedef unsigned long long zig_Builtin64; #endif static inline uint8_t zig_byte_swap_u8(uint8_t val, uint8_t bits) { return zig_wrap_u8(val >> (8 - bits), bits); } static inline int8_t zig_byte_swap_i8(int8_t val, uint8_t bits) { return zig_wrap_i8((int8_t)zig_byte_swap_u8((uint8_t)val, bits), bits); } static inline uint16_t zig_byte_swap_u16(uint16_t val, uint8_t bits) { uint16_t full_res; #if zig_has_builtin(bswap16) || defined(zig_gcc) full_res = __builtin_bswap16(val); #else full_res = (uint16_t)zig_byte_swap_u8((uint8_t)(val >> 0), 8) << 8 | (uint16_t)zig_byte_swap_u8((uint8_t)(val >> 8), 8) >> 0; #endif return zig_wrap_u16(full_res >> (16 - bits), bits); } static inline int16_t zig_byte_swap_i16(int16_t val, uint8_t bits) { return zig_wrap_i16((int16_t)zig_byte_swap_u16((uint16_t)val, bits), bits); } static inline uint32_t zig_byte_swap_u32(uint32_t val, uint8_t bits) { uint32_t full_res; #if zig_has_builtin(bswap32) || defined(zig_gcc) full_res = __builtin_bswap32(val); #else full_res = (uint32_t)zig_byte_swap_u16((uint16_t)(val >> 0), 16) << 16 | (uint32_t)zig_byte_swap_u16((uint16_t)(val >> 16), 16) >> 0; #endif return zig_wrap_u32(full_res >> (32 - bits), bits); } static inline int32_t zig_byte_swap_i32(int32_t val, uint8_t bits) { return zig_wrap_i32((int32_t)zig_byte_swap_u32((uint32_t)val, bits), bits); } static inline uint64_t zig_byte_swap_u64(uint64_t val, uint8_t bits) { uint64_t full_res; #if zig_has_builtin(bswap64) || defined(zig_gcc) full_res = __builtin_bswap64(val); #else full_res = (uint64_t)zig_byte_swap_u32((uint32_t)(val >> 0), 32) << 32 | (uint64_t)zig_byte_swap_u32((uint32_t)(val >> 32), 32) >> 0; #endif return zig_wrap_u64(full_res >> (64 - bits), bits); } static inline int64_t zig_byte_swap_i64(int64_t val, uint8_t bits) { return zig_wrap_i64((int64_t)zig_byte_swap_u64((uint64_t)val, bits), bits); } static inline uint8_t zig_bit_reverse_u8(uint8_t val, uint8_t bits) { uint8_t full_res; #if zig_has_builtin(bitreverse8) full_res = __builtin_bitreverse8(val); #else static uint8_t const lut[0x10] = { 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf }; full_res = lut[val >> 0 & 0xF] << 4 | lut[val >> 4 & 0xF] << 0; #endif return zig_wrap_u8(full_res >> (8 - bits), bits); } static inline int8_t zig_bit_reverse_i8(int8_t val, uint8_t bits) { return zig_wrap_i8((int8_t)zig_bit_reverse_u8((uint8_t)val, bits), bits); } static inline uint16_t zig_bit_reverse_u16(uint16_t val, uint8_t bits) { uint16_t full_res; #if zig_has_builtin(bitreverse16) full_res = __builtin_bitreverse16(val); #else full_res = (uint16_t)zig_bit_reverse_u8((uint8_t)(val >> 0), 8) << 8 | (uint16_t)zig_bit_reverse_u8((uint8_t)(val >> 8), 8) >> 0; #endif return zig_wrap_u16(full_res >> (16 - bits), bits); } static inline int16_t zig_bit_reverse_i16(int16_t val, uint8_t bits) { return zig_wrap_i16((int16_t)zig_bit_reverse_u16((uint16_t)val, bits), bits); } static inline uint32_t zig_bit_reverse_u32(uint32_t val, uint8_t bits) { uint32_t full_res; #if zig_has_builtin(bitreverse32) full_res = __builtin_bitreverse32(val); #else full_res = (uint32_t)zig_bit_reverse_u16((uint16_t)(val >> 0), 16) << 16 | (uint32_t)zig_bit_reverse_u16((uint16_t)(val >> 16), 16) >> 0; #endif return zig_wrap_u32(full_res >> (32 - bits), bits); } static inline int32_t zig_bit_reverse_i32(int32_t val, uint8_t bits) { return zig_wrap_i32((int32_t)zig_bit_reverse_u32((uint32_t)val, bits), bits); } static inline uint64_t zig_bit_reverse_u64(uint64_t val, uint8_t bits) { uint64_t full_res; #if zig_has_builtin(bitreverse64) full_res = __builtin_bitreverse64(val); #else full_res = (uint64_t)zig_bit_reverse_u32((uint32_t)(val >> 0), 32) << 32 | (uint64_t)zig_bit_reverse_u32((uint32_t)(val >> 32), 32) >> 0; #endif return zig_wrap_u64(full_res >> (64 - bits), bits); } static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) { return zig_wrap_i64((int64_t)zig_bit_reverse_u64((uint64_t)val, bits), bits); } #define zig_builtin_popcount_common(w) \ static inline uint8_t zig_popcount_i##w(int##w##_t val, uint8_t bits) { \ return zig_popcount_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(popcount) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_popcount(w) \ static inline uint8_t zig_popcount_u##w(uint##w##_t val, uint8_t bits) { \ (void)bits; \ return zig_builtin##w(popcount, val); \ } \ \ zig_builtin_popcount_common(w) #else #define zig_builtin_popcount(w) \ static inline uint8_t zig_popcount_u##w(uint##w##_t val, uint8_t bits) { \ (void)bits; \ uint##w##_t temp = val - ((val >> 1) & (UINT##w##_MAX / 3)); \ temp = (temp & (UINT##w##_MAX / 5)) + ((temp >> 2) & (UINT##w##_MAX / 5)); \ temp = (temp + (temp >> 4)) & (UINT##w##_MAX / 17); \ return temp * (UINT##w##_MAX / 255) >> (UINT8_C(w) - UINT8_C(8)); \ } \ \ zig_builtin_popcount_common(w) #endif zig_builtin_popcount(8) zig_builtin_popcount(16) zig_builtin_popcount(32) zig_builtin_popcount(64) #define zig_builtin_ctz_common(w) \ static inline uint8_t zig_ctz_i##w(int##w##_t val, uint8_t bits) { \ return zig_ctz_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(ctz) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_ctz(w) \ static inline uint8_t zig_ctz_u##w(uint##w##_t val, uint8_t bits) { \ if (val == 0) return bits; \ return zig_builtin##w(ctz, val); \ } \ \ zig_builtin_ctz_common(w) #else #define zig_builtin_ctz(w) \ static inline uint8_t zig_ctz_u##w(uint##w##_t val, uint8_t bits) { \ return zig_popcount_u##w(zig_not_u##w(val, bits) & zig_subw_u##w(val, 1, bits), bits); \ } \ \ zig_builtin_ctz_common(w) #endif zig_builtin_ctz(8) zig_builtin_ctz(16) zig_builtin_ctz(32) zig_builtin_ctz(64) #define zig_builtin_clz_common(w) \ static inline uint8_t zig_clz_i##w(int##w##_t val, uint8_t bits) { \ return zig_clz_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(clz) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_clz(w) \ static inline uint8_t zig_clz_u##w(uint##w##_t val, uint8_t bits) { \ if (val == 0) return bits; \ return zig_builtin##w(clz, val) - (zig_bitSizeOf(zig_Builtin##w) - bits); \ } \ \ zig_builtin_clz_common(w) #else #define zig_builtin_clz(w) \ static inline uint8_t zig_clz_u##w(uint##w##_t val, uint8_t bits) { \ return zig_ctz_u##w(zig_bit_reverse_u##w(val, bits), bits); \ } \ \ zig_builtin_clz_common(w) #endif zig_builtin_clz(8) zig_builtin_clz(16) zig_builtin_clz(32) zig_builtin_clz(64) /* ======================== 128-bit Integer Support ========================= */ #if !defined(zig_has_int128) # if defined(__SIZEOF_INT128__) # define zig_has_int128 1 # else # define zig_has_int128 0 # endif #endif #if zig_has_int128 typedef unsigned __int128 zig_u128; typedef signed __int128 zig_i128; #define zig_make_u128(hi, lo) ((zig_u128)(hi)<<64|(lo)) #define zig_make_i128(hi, lo) ((zig_i128)zig_make_u128(hi, lo)) #define zig_init_u128(hi, lo) zig_make_u128(hi, lo) #define zig_init_i128(hi, lo) zig_make_i128(hi, lo) #define zig_hi_u128(val) ((uint64_t)((val) >> 64)) #define zig_lo_u128(val) ((uint64_t)((val) >> 0)) #define zig_hi_i128(val) (( int64_t)((val) >> 64)) #define zig_lo_i128(val) ((uint64_t)((val) >> 0)) #define zig_bitCast_u128(val) ((zig_u128)(val)) #define zig_bitCast_i128(val) ((zig_i128)(val)) #define zig_cmp_int128(Type) \ static inline int32_t zig_cmp_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (lhs > rhs) - (lhs < rhs); \ } #define zig_bit_int128(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return lhs operator rhs; \ } #else /* zig_has_int128 */ #if zig_little_endian typedef struct { zig_align(16) uint64_t lo; uint64_t hi; } zig_u128; typedef struct { zig_align(16) uint64_t lo; int64_t hi; } zig_i128; #else typedef struct { zig_align(16) uint64_t hi; uint64_t lo; } zig_u128; typedef struct { zig_align(16) int64_t hi; uint64_t lo; } zig_i128; #endif #define zig_make_u128(hi, lo) ((zig_u128){ .h##i = (hi), .l##o = (lo) }) #define zig_make_i128(hi, lo) ((zig_i128){ .h##i = (hi), .l##o = (lo) }) #if defined(zig_msvc) /* MSVC doesn't allow struct literals in constant expressions */ #define zig_init_u128(hi, lo) { .h##i = (hi), .l##o = (lo) } #define zig_init_i128(hi, lo) { .h##i = (hi), .l##o = (lo) } #else /* But non-MSVC doesn't like the unprotected commas */ #define zig_init_u128(hi, lo) zig_make_u128(hi, lo) #define zig_init_i128(hi, lo) zig_make_i128(hi, lo) #endif #define zig_hi_u128(val) ((val).hi) #define zig_lo_u128(val) ((val).lo) #define zig_hi_i128(val) ((val).hi) #define zig_lo_i128(val) ((val).lo) #define zig_bitCast_u128(val) zig_make_u128((uint64_t)(val).hi, (val).lo) #define zig_bitCast_i128(val) zig_make_i128(( int64_t)(val).hi, (val).lo) #define zig_cmp_int128(Type) \ static inline int32_t zig_cmp_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (lhs.hi == rhs.hi) \ ? (lhs.lo > rhs.lo) - (lhs.lo < rhs.lo) \ : (lhs.hi > rhs.hi) - (lhs.hi < rhs.hi); \ } #define zig_bit_int128(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (zig_##Type){ .hi = lhs.hi operator rhs.hi, .lo = lhs.lo operator rhs.lo }; \ } #endif /* zig_has_int128 */ #define zig_minInt_u128 zig_make_u128(zig_minInt_u64, zig_minInt_u64) #define zig_maxInt_u128 zig_make_u128(zig_maxInt_u64, zig_maxInt_u64) #define zig_minInt_i128 zig_make_i128(zig_minInt_i64, zig_minInt_u64) #define zig_maxInt_i128 zig_make_i128(zig_maxInt_i64, zig_maxInt_u64) zig_cmp_int128(u128) zig_cmp_int128(i128) zig_bit_int128(u128, and, &) zig_bit_int128(i128, and, &) zig_bit_int128(u128, or, |) zig_bit_int128(i128, or, |) zig_bit_int128(u128, xor, ^) zig_bit_int128(i128, xor, ^) static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs); #if zig_has_int128 static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) { return val ^ zig_maxInt_u(128, bits); } static inline zig_i128 zig_not_i128(zig_i128 val, uint8_t bits) { (void)bits; return ~val; } static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs) { return lhs >> rhs; } static inline zig_u128 zig_shl_u128(zig_u128 lhs, uint8_t rhs) { return lhs << rhs; } static inline zig_i128 zig_shr_i128(zig_i128 lhs, uint8_t rhs) { zig_i128 sign_mask = lhs < zig_make_i128(0, 0) ? -zig_make_i128(0, 1) : zig_make_i128(0, 0); return ((lhs ^ sign_mask) >> rhs) ^ sign_mask; } static inline zig_i128 zig_shl_i128(zig_i128 lhs, uint8_t rhs) { return lhs << rhs; } static inline zig_u128 zig_add_u128(zig_u128 lhs, zig_u128 rhs) { return lhs + rhs; } static inline zig_i128 zig_add_i128(zig_i128 lhs, zig_i128 rhs) { return lhs + rhs; } static inline zig_u128 zig_sub_u128(zig_u128 lhs, zig_u128 rhs) { return lhs - rhs; } static inline zig_i128 zig_sub_i128(zig_i128 lhs, zig_i128 rhs) { return lhs - rhs; } static inline zig_u128 zig_mul_u128(zig_u128 lhs, zig_u128 rhs) { return lhs * rhs; } static inline zig_i128 zig_mul_i128(zig_i128 lhs, zig_i128 rhs) { return lhs * rhs; } static inline zig_u128 zig_div_trunc_u128(zig_u128 lhs, zig_u128 rhs) { return lhs / rhs; } static inline zig_i128 zig_div_trunc_i128(zig_i128 lhs, zig_i128 rhs) { return lhs / rhs; } static inline zig_u128 zig_rem_u128(zig_u128 lhs, zig_u128 rhs) { return lhs % rhs; } static inline zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { return lhs % rhs; } #else /* zig_has_int128 */ static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) { return (zig_u128){ .hi = zig_not_u64(val.hi, bits - UINT8_C(64)), .lo = zig_not_u64(val.lo, UINT8_C(64)) }; } static inline zig_i128 zig_not_i128(zig_i128 val, uint8_t bits) { return (zig_i128){ .hi = zig_not_i64(val.hi, bits - UINT8_C(64)), .lo = zig_not_u64(val.lo, UINT8_C(64)) }; } static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_u128){ .hi = zig_minInt_u64, .lo = lhs.hi >> (rhs - UINT8_C(64)) }; return (zig_u128){ .hi = lhs.hi >> rhs, .lo = lhs.hi << (UINT8_C(64) - rhs) | lhs.lo >> rhs }; } static inline zig_u128 zig_shl_u128(zig_u128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_u128){ .hi = lhs.lo << (rhs - UINT8_C(64)), .lo = zig_minInt_u64 }; return (zig_u128){ .hi = lhs.hi << rhs | lhs.lo >> (UINT8_C(64) - rhs), .lo = lhs.lo << rhs }; } static inline zig_i128 zig_shr_i128(zig_i128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_i128){ .hi = zig_shr_i64(lhs.hi, 63), .lo = zig_shr_i64(lhs.hi, (rhs - UINT8_C(64))) }; return (zig_i128){ .hi = zig_shr_i64(lhs.hi, rhs), .lo = lhs.lo >> rhs | (uint64_t)lhs.hi << (UINT8_C(64) - rhs) }; } static inline zig_i128 zig_shl_i128(zig_i128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_i128){ .hi = lhs.lo << (rhs - UINT8_C(64)), .lo = zig_minInt_u64 }; return (zig_i128){ .hi = lhs.hi << rhs | lhs.lo >> (UINT8_C(64) - rhs), .lo = lhs.lo << rhs }; } static inline zig_u128 zig_add_u128(zig_u128 lhs, zig_u128 rhs) { zig_u128 res; res.hi = lhs.hi + rhs.hi + zig_addo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_i128 zig_add_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 res; res.hi = lhs.hi + rhs.hi + zig_addo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_u128 zig_sub_u128(zig_u128 lhs, zig_u128 rhs) { zig_u128 res; res.hi = lhs.hi - rhs.hi - zig_subo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_i128 zig_sub_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 res; res.hi = lhs.hi - rhs.hi - zig_subo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } zig_extern zig_i128 __multi3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_mul_i128(zig_i128 lhs, zig_i128 rhs) { return __multi3(lhs, rhs); } static zig_u128 zig_mul_u128(zig_u128 lhs, zig_u128 rhs) { return zig_bitCast_u128(zig_mul_i128(zig_bitCast_i128(lhs), zig_bitCast_i128(rhs))); } zig_extern zig_u128 __udivti3(zig_u128 lhs, zig_u128 rhs); static zig_u128 zig_div_trunc_u128(zig_u128 lhs, zig_u128 rhs) { return __udivti3(lhs, rhs); } zig_extern zig_i128 __divti3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_div_trunc_i128(zig_i128 lhs, zig_i128 rhs) { return __divti3(lhs, rhs); } zig_extern zig_u128 __umodti3(zig_u128 lhs, zig_u128 rhs); static zig_u128 zig_rem_u128(zig_u128 lhs, zig_u128 rhs) { return __umodti3(lhs, rhs); } zig_extern zig_i128 __modti3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { return __modti3(lhs, rhs); } #endif /* zig_has_int128 */ #define zig_div_floor_u128 zig_div_trunc_u128 static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 rem = zig_rem_i128(lhs, rhs); int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); return zig_add_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(mask, (uint64_t)mask)); } #define zig_mod_u128 zig_rem_u128 static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 rem = zig_rem_i128(lhs, rhs); int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); return zig_add_i128(rem, zig_and_i128(rhs, zig_make_i128(mask, (uint64_t)mask))); } static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) { return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs; } static inline zig_i128 zig_min_i128(zig_i128 lhs, zig_i128 rhs) { return zig_cmp_i128(lhs, rhs) < INT32_C(0) ? lhs : rhs; } static inline zig_u128 zig_max_u128(zig_u128 lhs, zig_u128 rhs) { return zig_cmp_u128(lhs, rhs) > INT32_C(0) ? lhs : rhs; } static inline zig_i128 zig_max_i128(zig_i128 lhs, zig_i128 rhs) { return zig_cmp_i128(lhs, rhs) > INT32_C(0) ? lhs : rhs; } static inline zig_u128 zig_wrap_u128(zig_u128 val, uint8_t bits) { return zig_and_u128(val, zig_maxInt_u(128, bits)); } static inline zig_i128 zig_wrap_i128(zig_i128 val, uint8_t bits) { if (bits > UINT8_C(64)) return zig_make_i128(zig_wrap_i64(zig_hi_i128(val), bits - UINT8_C(64)), zig_lo_i128(val)); int64_t lo = zig_wrap_i64((int64_t)zig_lo_i128(val), bits); return zig_make_i128(zig_shr_i64(lo, 63), (uint64_t)lo); } static inline zig_u128 zig_shlw_u128(zig_u128 lhs, uint8_t rhs, uint8_t bits) { return zig_wrap_u128(zig_shl_u128(lhs, rhs), bits); } static inline zig_i128 zig_shlw_i128(zig_i128 lhs, uint8_t rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_shl_u128(zig_bitCast_u128(lhs), rhs)), bits); } static inline zig_u128 zig_addw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_add_u128(lhs, rhs), bits); } static inline zig_i128 zig_addw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_add_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_subw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_sub_u128(lhs, rhs), bits); } static inline zig_i128 zig_subw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_sub_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_mulw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_mul_u128(lhs, rhs), bits); } static inline zig_i128 zig_mulw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_mul_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_abs_i128(zig_i128 val) { zig_i128 tmp = zig_shr_i128(val, 127); return zig_bitCast_u128(zig_sub_i128(zig_xor_i128(val, tmp), tmp)); } #if zig_has_int128 static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) zig_u128 full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_addw_u128(lhs, rhs, bits); return *res < lhs; #endif } zig_extern zig_i128 __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) zig_i128 full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __addoti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) zig_u128 full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_subw_u128(lhs, rhs, bits); return *res > lhs; #endif } zig_extern zig_i128 __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) zig_i128 full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } static inline bool zig_mulo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) zig_u128 full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_mulw_u128(lhs, rhs, bits); return rhs != zig_make_u128(0, 0) && lhs > zig_maxInt_u(128, bits) / rhs; #endif } zig_extern zig_i128 __muloti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_mulo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) zig_i128 full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __muloti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } #else /* zig_has_int128 */ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { uint64_t hi; bool overflow = zig_addo_u64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_addo_u64(&res->hi, hi, zig_addo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int64_t hi; bool overflow = zig_addo_i64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_addo_i64(&res->hi, hi, zig_addo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { uint64_t hi; bool overflow = zig_subo_u64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_subo_u64(&res->hi, hi, zig_subo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int64_t hi; bool overflow = zig_subo_i64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_subo_i64(&res->hi, hi, zig_subo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_mulo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { *res = zig_mulw_u128(lhs, rhs, bits); return zig_cmp_u128(*res, zig_make_u128(0, 0)) != INT32_C(0) && zig_cmp_u128(lhs, zig_div_trunc_u128(zig_maxInt_u(128, bits), rhs)) > INT32_C(0); } zig_extern zig_i128 __muloti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_mulo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int overflow_int; zig_i128 full_res = __muloti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0 || zig_cmp_i128(full_res, zig_minInt_i(128, bits)) < INT32_C(0) || zig_cmp_i128(full_res, zig_maxInt_i(128, bits)) > INT32_C(0); *res = zig_wrap_i128(full_res, bits); return overflow; } #endif /* zig_has_int128 */ static inline bool zig_shlo_u128(zig_u128 *res, zig_u128 lhs, uint8_t rhs, uint8_t bits) { *res = zig_shlw_u128(lhs, rhs, bits); return zig_cmp_u128(lhs, zig_shr_u128(zig_maxInt_u(128, bits), rhs)) > INT32_C(0); } static inline bool zig_shlo_i128(zig_i128 *res, zig_i128 lhs, uint8_t rhs, uint8_t bits) { *res = zig_shlw_i128(lhs, rhs, bits); zig_i128 mask = zig_bitCast_i128(zig_shl_u128(zig_maxInt_u128, bits - rhs - UINT8_C(1))); return zig_cmp_i128(zig_and_i128(lhs, mask), zig_make_i128(0, 0)) != INT32_C(0) && zig_cmp_i128(zig_and_i128(lhs, mask), mask) != INT32_C(0); } static inline zig_u128 zig_shls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; if (zig_cmp_u128(rhs, zig_make_u128(0, bits)) >= INT32_C(0)) return zig_cmp_u128(lhs, zig_make_u128(0, 0)) != INT32_C(0) ? zig_maxInt_u(128, bits) : lhs; return zig_shlo_u128(&res, lhs, (uint8_t)zig_lo_u128(rhs), bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_shls_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (zig_cmp_u128(zig_bitCast_u128(rhs), zig_make_u128(0, bits)) < INT32_C(0) && !zig_shlo_i128(&res, lhs, (uint8_t)zig_lo_i128(rhs), bits)) return res; return zig_cmp_i128(lhs, zig_make_i128(0, 0)) < INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_adds_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_addo_u128(&res, lhs, rhs, bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_adds_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_addo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(res, zig_make_i128(0, 0)) >= INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_subs_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_subo_u128(&res, lhs, rhs, bits) ? zig_minInt_u(128, bits) : res; } static inline zig_i128 zig_subs_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_subo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(res, zig_make_i128(0, 0)) >= INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_muls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_mulo_u128(&res, lhs, rhs, bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_muls_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_mulo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(zig_xor_i128(lhs, rhs), zig_make_i128(0, 0)) < INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline uint8_t zig_clz_u128(zig_u128 val, uint8_t bits) { if (bits <= UINT8_C(64)) return zig_clz_u64(zig_lo_u128(val), bits); if (zig_hi_u128(val) != 0) return zig_clz_u64(zig_hi_u128(val), bits - UINT8_C(64)); return zig_clz_u64(zig_lo_u128(val), UINT8_C(64)) + (bits - UINT8_C(64)); } static inline uint8_t zig_clz_i128(zig_i128 val, uint8_t bits) { return zig_clz_u128(zig_bitCast_u128(val), bits); } static inline uint8_t zig_ctz_u128(zig_u128 val, uint8_t bits) { if (zig_lo_u128(val) != 0) return zig_ctz_u64(zig_lo_u128(val), UINT8_C(64)); return zig_ctz_u64(zig_hi_u128(val), bits - UINT8_C(64)) + UINT8_C(64); } static inline uint8_t zig_ctz_i128(zig_i128 val, uint8_t bits) { return zig_ctz_u128(zig_bitCast_u128(val), bits); } static inline uint8_t zig_popcount_u128(zig_u128 val, uint8_t bits) { return zig_popcount_u64(zig_hi_u128(val), bits - UINT8_C(64)) + zig_popcount_u64(zig_lo_u128(val), UINT8_C(64)); } static inline uint8_t zig_popcount_i128(zig_i128 val, uint8_t bits) { return zig_popcount_u128(zig_bitCast_u128(val), bits); } static inline zig_u128 zig_byte_swap_u128(zig_u128 val, uint8_t bits) { zig_u128 full_res; #if zig_has_builtin(bswap128) full_res = __builtin_bswap128(val); #else full_res = zig_make_u128(zig_byte_swap_u64(zig_lo_u128(val), UINT8_C(64)), zig_byte_swap_u64(zig_hi_u128(val), UINT8_C(64))); #endif return zig_shr_u128(full_res, UINT8_C(128) - bits); } static inline zig_i128 zig_byte_swap_i128(zig_i128 val, uint8_t bits) { return zig_bitCast_i128(zig_byte_swap_u128(zig_bitCast_u128(val), bits)); } static inline zig_u128 zig_bit_reverse_u128(zig_u128 val, uint8_t bits) { return zig_shr_u128(zig_make_u128(zig_bit_reverse_u64(zig_lo_u128(val), UINT8_C(64)), zig_bit_reverse_u64(zig_hi_u128(val), UINT8_C(64))), UINT8_C(128) - bits); } static inline zig_i128 zig_bit_reverse_i128(zig_i128 val, uint8_t bits) { return zig_bitCast_i128(zig_bit_reverse_u128(zig_bitCast_u128(val), bits)); } /* ========================== Big Integer Support =========================== */ static inline uint16_t zig_int_bytes(uint16_t bits) { uint16_t bytes = (bits + CHAR_BIT - 1) / CHAR_BIT; uint16_t alignment = ZIG_TARGET_MAX_INT_ALIGNMENT; while (alignment / 2 >= bytes) alignment /= 2; return (bytes + alignment - 1) / alignment * alignment; } static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; bool do_signed = is_signed; uint16_t remaining_bytes = zig_int_bytes(bits); #if zig_little_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { int32_t limb_cmp; #if zig_little_endian byte_offset -= 128 / CHAR_BIT; #endif if (do_signed) { zig_i128 lhs_limb; zig_i128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_cmp = zig_cmp_i128(lhs_limb, rhs_limb); do_signed = false; } else { zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_cmp = zig_cmp_u128(lhs_limb, rhs_limb); } if (limb_cmp != 0) return limb_cmp; remaining_bytes -= 128 / CHAR_BIT; #if zig_big_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_little_endian byte_offset -= 64 / CHAR_BIT; #endif if (do_signed) { int64_t lhs_limb; int64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 64 / CHAR_BIT; #if zig_big_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_little_endian byte_offset -= 32 / CHAR_BIT; #endif if (do_signed) { int32_t lhs_limb; int32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 32 / CHAR_BIT; #if zig_big_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_little_endian byte_offset -= 16 / CHAR_BIT; #endif if (do_signed) { int16_t lhs_limb; int16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 16 / CHAR_BIT; #if zig_big_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_little_endian byte_offset -= 8 / CHAR_BIT; #endif if (do_signed) { int8_t lhs_limb; int8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 8 / CHAR_BIT; #if zig_big_endian byte_offset += 8 / CHAR_BIT; #endif } return 0; } static inline void zig_and_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline void zig_or_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline void zig_xor_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline bool zig_addo_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint8_t top_bits = (uint8_t)(remaining_bytes * 8 - bits); bool overflow = false; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { uint8_t limb_bits = 128 - (remaining_bytes == 128 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif if (remaining_bytes == 128 / CHAR_BIT && is_signed) { zig_i128 res_limb; zig_i128 tmp_limb; zig_i128 lhs_limb; zig_i128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i128(&res_limb, tmp_limb, zig_make_i128(INT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { zig_u128 res_limb; zig_u128 tmp_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u128(&res_limb, tmp_limb, zig_make_u128(UINT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { uint8_t limb_bits = 64 - (remaining_bytes == 64 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif if (remaining_bytes == 64 / CHAR_BIT && is_signed) { int64_t res_limb; int64_t tmp_limb; int64_t lhs_limb; int64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i64(&res_limb, tmp_limb, overflow ? INT64_C(1) : INT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint64_t res_limb; uint64_t tmp_limb; uint64_t lhs_limb; uint64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u64(&res_limb, tmp_limb, overflow ? UINT64_C(1) : UINT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { uint8_t limb_bits = 32 - (remaining_bytes == 32 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif if (remaining_bytes == 32 / CHAR_BIT && is_signed) { int32_t res_limb; int32_t tmp_limb; int32_t lhs_limb; int32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i32(&res_limb, tmp_limb, overflow ? INT32_C(1) : INT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint32_t res_limb; uint32_t tmp_limb; uint32_t lhs_limb; uint32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u32(&res_limb, tmp_limb, overflow ? UINT32_C(1) : UINT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { uint8_t limb_bits = 16 - (remaining_bytes == 16 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif if (remaining_bytes == 16 / CHAR_BIT && is_signed) { int16_t res_limb; int16_t tmp_limb; int16_t lhs_limb; int16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i16(&res_limb, tmp_limb, overflow ? INT16_C(1) : INT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint16_t res_limb; uint16_t tmp_limb; uint16_t lhs_limb; uint16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u16(&res_limb, tmp_limb, overflow ? UINT16_C(1) : UINT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t limb_bits = 8 - (remaining_bytes == 8 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif if (remaining_bytes == 8 / CHAR_BIT && is_signed) { int8_t res_limb; int8_t tmp_limb; int8_t lhs_limb; int8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i8(&res_limb, tmp_limb, overflow ? INT8_C(1) : INT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint8_t res_limb; uint8_t tmp_limb; uint8_t lhs_limb; uint8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u8(&res_limb, tmp_limb, overflow ? UINT8_C(1) : UINT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return overflow; } static inline bool zig_subo_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint8_t top_bits = (uint8_t)(remaining_bytes * 8 - bits); bool overflow = false; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { uint8_t limb_bits = 128 - (remaining_bytes == 128 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif if (remaining_bytes == 128 / CHAR_BIT && is_signed) { zig_i128 res_limb; zig_i128 tmp_limb; zig_i128 lhs_limb; zig_i128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i128(&res_limb, tmp_limb, zig_make_i128(INT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { zig_u128 res_limb; zig_u128 tmp_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u128(&res_limb, tmp_limb, zig_make_u128(UINT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { uint8_t limb_bits = 64 - (remaining_bytes == 64 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif if (remaining_bytes == 64 / CHAR_BIT && is_signed) { int64_t res_limb; int64_t tmp_limb; int64_t lhs_limb; int64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i64(&res_limb, tmp_limb, overflow ? INT64_C(1) : INT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint64_t res_limb; uint64_t tmp_limb; uint64_t lhs_limb; uint64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u64(&res_limb, tmp_limb, overflow ? UINT64_C(1) : UINT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { uint8_t limb_bits = 32 - (remaining_bytes == 32 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif if (remaining_bytes == 32 / CHAR_BIT && is_signed) { int32_t res_limb; int32_t tmp_limb; int32_t lhs_limb; int32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i32(&res_limb, tmp_limb, overflow ? INT32_C(1) : INT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint32_t res_limb; uint32_t tmp_limb; uint32_t lhs_limb; uint32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u32(&res_limb, tmp_limb, overflow ? UINT32_C(1) : UINT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { uint8_t limb_bits = 16 - (remaining_bytes == 16 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif if (remaining_bytes == 16 / CHAR_BIT && is_signed) { int16_t res_limb; int16_t tmp_limb; int16_t lhs_limb; int16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i16(&res_limb, tmp_limb, overflow ? INT16_C(1) : INT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint16_t res_limb; uint16_t tmp_limb; uint16_t lhs_limb; uint16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u16(&res_limb, tmp_limb, overflow ? UINT16_C(1) : UINT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t limb_bits = 8 - (remaining_bytes == 8 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif if (remaining_bytes == 8 / CHAR_BIT && is_signed) { int8_t res_limb; int8_t tmp_limb; int8_t lhs_limb; int8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i8(&res_limb, tmp_limb, overflow ? INT8_C(1) : INT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint8_t res_limb; uint8_t tmp_limb; uint8_t lhs_limb; uint8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u8(&res_limb, tmp_limb, overflow ? UINT8_C(1) : UINT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return overflow; } static inline void zig_addw_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { (void)zig_addo_big(res, lhs, rhs, is_signed, bits); } static inline void zig_subw_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { (void)zig_subo_big(res, lhs, rhs, is_signed, bits); } zig_extern void __udivei4(uint32_t *res, const uint32_t *lhs, const uint32_t *rhs, uintptr_t bits); static inline void zig_div_trunc_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { __udivei4(res, lhs, rhs, bits); return; } zig_trap(); } static inline void zig_div_floor_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { zig_div_trunc_big(res, lhs, rhs, is_signed, bits); return; } zig_trap(); } zig_extern void __umodei4(uint32_t *res, const uint32_t *lhs, const uint32_t *rhs, uintptr_t bits); static inline void zig_rem_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { __umodei4(res, lhs, rhs, bits); return; } zig_trap(); } static inline void zig_mod_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { zig_rem_big(res, lhs, rhs, is_signed, bits); return; } zig_trap(); } static inline uint16_t zig_clz_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t skip_bits = remaining_bytes * 8 - bits; uint16_t total_lz = 0; uint16_t limb_lz; (void)is_signed; #if zig_little_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_little_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u128(val_limb, 128 - skip_bits); } total_lz += limb_lz; if (limb_lz < 128 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 128 / CHAR_BIT; #if zig_big_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_little_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u64(val_limb, 64 - skip_bits); } total_lz += limb_lz; if (limb_lz < 64 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 64 / CHAR_BIT; #if zig_big_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_little_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u32(val_limb, 32 - skip_bits); } total_lz += limb_lz; if (limb_lz < 32 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 32 / CHAR_BIT; #if zig_big_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_little_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u16(val_limb, 16 - skip_bits); } total_lz += limb_lz; if (limb_lz < 16 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 16 / CHAR_BIT; #if zig_big_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_little_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u8(val_limb, 8 - skip_bits); } total_lz += limb_lz; if (limb_lz < 8 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 8 / CHAR_BIT; #if zig_big_endian byte_offset += 8 / CHAR_BIT; #endif } return total_lz; } static inline uint16_t zig_ctz_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t total_tz = 0; uint16_t limb_tz; (void)is_signed; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u128(val_limb, 128); } total_tz += limb_tz; if (limb_tz < 128) return total_tz; remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u64(val_limb, 64); } total_tz += limb_tz; if (limb_tz < 64) return total_tz; remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u32(val_limb, 32); } total_tz += limb_tz; if (limb_tz < 32) return total_tz; remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u16(val_limb, 16); } total_tz += limb_tz; if (limb_tz < 16) return total_tz; remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u8(val_limb, 8); } total_tz += limb_tz; if (limb_tz < 8) return total_tz; remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return total_tz; } static inline uint16_t zig_popcount_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t total_pc = 0; (void)is_signed; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u128(val_limb, 128); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u64(val_limb, 64); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u32(val_limb, 32); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc = zig_popcount_u16(val_limb, 16); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc = zig_popcount_u8(val_limb, 8); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return total_pc; } /* ========================= Floating Point Support ========================= */ #ifndef __STDC_WANT_IEC_60559_TYPES_EXT__ #define __STDC_WANT_IEC_60559_TYPES_EXT__ #endif #include #if defined(zig_msvc) float __cdecl nanf(char const* input); double __cdecl nan(char const* input); long double __cdecl nanl(char const* input); #define zig_msvc_flt_inf ((double)(1e+300 * 1e+300)) #define zig_msvc_flt_inff ((float)(1e+300 * 1e+300)) #define zig_msvc_flt_infl ((long double)(1e+300 * 1e+300)) #define zig_msvc_flt_nan ((double)(zig_msvc_flt_inf * 0.f)) #define zig_msvc_flt_nanf ((float)(zig_msvc_flt_inf * 0.f)) #define zig_msvc_flt_nanl ((long double)(zig_msvc_flt_inf * 0.f)) #define __builtin_nan(str) nan(str) #define __builtin_nanf(str) nanf(str) #define __builtin_nanl(str) nanl(str) #define __builtin_inf() zig_msvc_flt_inf #define __builtin_inff() zig_msvc_flt_inff #define __builtin_infl() zig_msvc_flt_infl #endif #if (zig_has_builtin(nan) && zig_has_builtin(nans) && zig_has_builtin(inf)) || defined(zig_gcc) #define zig_make_special_f16(sign, name, arg, repr) sign zig_make_f16 (__builtin_##name, )(arg) #define zig_make_special_f32(sign, name, arg, repr) sign zig_make_f32 (__builtin_##name, )(arg) #define zig_make_special_f64(sign, name, arg, repr) sign zig_make_f64 (__builtin_##name, )(arg) #define zig_make_special_f80(sign, name, arg, repr) sign zig_make_f80 (__builtin_##name, )(arg) #define zig_make_special_f128(sign, name, arg, repr) sign zig_make_f128(__builtin_##name, )(arg) #else #define zig_make_special_f16(sign, name, arg, repr) zig_bitCast_f16 (repr) #define zig_make_special_f32(sign, name, arg, repr) zig_bitCast_f32 (repr) #define zig_make_special_f64(sign, name, arg, repr) zig_bitCast_f64 (repr) #define zig_make_special_f80(sign, name, arg, repr) zig_bitCast_f80 (repr) #define zig_make_special_f128(sign, name, arg, repr) zig_bitCast_f128(repr) #endif #define zig_has_f16 1 #define zig_libc_name_f16(name) __##name##h #define zig_init_special_f16(sign, name, arg, repr) zig_make_special_f16(sign, name, arg, repr) #if FLT_MANT_DIG == 11 typedef float zig_f16; #define zig_make_f16(fp, repr) fp##f #elif DBL_MANT_DIG == 11 typedef double zig_f16; #define zig_make_f16(fp, repr) fp #elif LDBL_MANT_DIG == 11 typedef long double zig_f16; #define zig_make_f16(fp, repr) fp##l #elif FLT16_MANT_DIG == 11 && (zig_has_builtin(inff16) || defined(zig_gcc)) typedef _Float16 zig_f16; #define zig_make_f16(fp, repr) fp##f16 #elif defined(__SIZEOF_FP16__) typedef __fp16 zig_f16; #define zig_make_f16(fp, repr) fp##f16 #else #undef zig_has_f16 #define zig_has_f16 0 #define zig_repr_f16 u16 typedef uint16_t zig_f16; #define zig_make_f16(fp, repr) repr #undef zig_make_special_f16 #define zig_make_special_f16(sign, name, arg, repr) repr #undef zig_init_special_f16 #define zig_init_special_f16(sign, name, arg, repr) repr #endif #if defined(zig_darwin) && defined(zig_x86) typedef uint16_t zig_compiler_rt_f16; #else typedef zig_f16 zig_compiler_rt_f16; #endif #define zig_has_f32 1 #define zig_libc_name_f32(name) name##f #if defined(zig_msvc) #define zig_init_special_f32(sign, name, arg, repr) sign zig_make_f32(zig_msvc_flt_##name, ) #else #define zig_init_special_f32(sign, name, arg, repr) zig_make_special_f32(sign, name, arg, repr) #endif #if FLT_MANT_DIG == 24 typedef float zig_f32; #define zig_make_f32(fp, repr) fp##f #elif DBL_MANT_DIG == 24 typedef double zig_f32; #define zig_make_f32(fp, repr) fp #elif LDBL_MANT_DIG == 24 typedef long double zig_f32; #define zig_make_f32(fp, repr) fp##l #elif FLT32_MANT_DIG == 24 typedef _Float32 zig_f32; #define zig_make_f32(fp, repr) fp##f32 #else #undef zig_has_f32 #define zig_has_f32 0 #define zig_repr_f32 u32 typedef uint32_t zig_f32; #define zig_make_f32(fp, repr) repr #undef zig_make_special_f32 #define zig_make_special_f32(sign, name, arg, repr) repr #undef zig_init_special_f32 #define zig_init_special_f32(sign, name, arg, repr) repr #endif #define zig_has_f64 1 #define zig_libc_name_f64(name) name #if defined(zig_msvc) #define zig_init_special_f64(sign, name, arg, repr) sign zig_make_f64(zig_msvc_flt_##name, ) #else #define zig_init_special_f64(sign, name, arg, repr) zig_make_special_f64(sign, name, arg, repr) #endif #if FLT_MANT_DIG == 53 typedef float zig_f64; #define zig_make_f64(fp, repr) fp##f #elif DBL_MANT_DIG == 53 typedef double zig_f64; #define zig_make_f64(fp, repr) fp #elif LDBL_MANT_DIG == 53 typedef long double zig_f64; #define zig_make_f64(fp, repr) fp##l #elif FLT64_MANT_DIG == 53 typedef _Float64 zig_f64; #define zig_make_f64(fp, repr) fp##f64 #elif FLT32X_MANT_DIG == 53 typedef _Float32x zig_f64; #define zig_make_f64(fp, repr) fp##f32x #else #undef zig_has_f64 #define zig_has_f64 0 #define zig_repr_f64 u64 typedef uint64_t zig_f64; #define zig_make_f64(fp, repr) repr #undef zig_make_special_f64 #define zig_make_special_f64(sign, name, arg, repr) repr #undef zig_init_special_f64 #define zig_init_special_f64(sign, name, arg, repr) repr #endif #define zig_has_f80 1 #define zig_libc_name_f80(name) __##name##x #define zig_init_special_f80(sign, name, arg, repr) zig_make_special_f80(sign, name, arg, repr) #if FLT_MANT_DIG == 64 typedef float zig_f80; #define zig_make_f80(fp, repr) fp##f #elif DBL_MANT_DIG == 64 typedef double zig_f80; #define zig_make_f80(fp, repr) fp #elif LDBL_MANT_DIG == 64 typedef long double zig_f80; #define zig_make_f80(fp, repr) fp##l #elif FLT80_MANT_DIG == 64 typedef _Float80 zig_f80; #define zig_make_f80(fp, repr) fp##f80 #elif FLT64X_MANT_DIG == 64 typedef _Float64x zig_f80; #define zig_make_f80(fp, repr) fp##f64x #elif defined(__SIZEOF_FLOAT80__) typedef __float80 zig_f80; #define zig_make_f80(fp, repr) fp##l #else #undef zig_has_f80 #define zig_has_f80 0 #define zig_repr_f80 u128 typedef zig_u128 zig_f80; #define zig_make_f80(fp, repr) repr #undef zig_make_special_f80 #define zig_make_special_f80(sign, name, arg, repr) repr #undef zig_init_special_f80 #define zig_init_special_f80(sign, name, arg, repr) repr #endif #if defined(zig_gcc) && defined(zig_x86) #define zig_f128_has_miscompilations 1 #else #define zig_f128_has_miscompilations 0 #endif #define zig_has_f128 1 #define zig_libc_name_f128(name) name##q #define zig_init_special_f128(sign, name, arg, repr) zig_make_special_f128(sign, name, arg, repr) #if !zig_f128_has_miscompilations && FLT_MANT_DIG == 113 typedef float zig_f128; #define zig_make_f128(fp, repr) fp##f #elif !zig_f128_has_miscompilations && DBL_MANT_DIG == 113 typedef double zig_f128; #define zig_make_f128(fp, repr) fp #elif !zig_f128_has_miscompilations && LDBL_MANT_DIG == 113 typedef long double zig_f128; #define zig_make_f128(fp, repr) fp##l #elif !zig_f128_has_miscompilations && FLT128_MANT_DIG == 113 typedef _Float128 zig_f128; #define zig_make_f128(fp, repr) fp##f128 #elif !zig_f128_has_miscompilations && FLT64X_MANT_DIG == 113 typedef _Float64x zig_f128; #define zig_make_f128(fp, repr) fp##f64x #elif !zig_f128_has_miscompilations && defined(__SIZEOF_FLOAT128__) typedef __float128 zig_f128; #define zig_make_f128(fp, repr) fp##q #undef zig_make_special_f128 #define zig_make_special_f128(sign, name, arg, repr) sign __builtin_##name##f128(arg) #else #undef zig_has_f128 #define zig_has_f128 0 #undef zig_make_special_f128 #undef zig_init_special_f128 #if defined(zig_darwin) || defined(zig_aarch64) typedef __attribute__((__vector_size__(2 * sizeof(uint64_t)))) uint64_t zig_v2u64; zig_basic_operator(zig_v2u64, xor_v2u64, ^) #define zig_repr_f128 v2u64 typedef zig_v2u64 zig_f128; #define zig_make_f128_zig_make_u128(hi, lo) (zig_f128){ lo, hi } #define zig_make_f128_zig_init_u128 zig_make_f128_zig_make_u128 #define zig_make_f128(fp, repr) zig_make_f128_##repr #define zig_make_special_f128(sign, name, arg, repr) zig_make_f128_##repr #define zig_init_special_f128(sign, name, arg, repr) zig_make_f128_##repr #else #define zig_repr_f128 u128 typedef zig_u128 zig_f128; #define zig_make_f128(fp, repr) repr #define zig_make_special_f128(sign, name, arg, repr) repr #define zig_init_special_f128(sign, name, arg, repr) repr #endif #endif #if !defined(zig_msvc) && defined(ZIG_TARGET_ABI_MSVC) /* Emulate msvc abi on a gnu compiler */ typedef zig_f64 zig_c_longdouble; #elif defined(zig_msvc) && !defined(ZIG_TARGET_ABI_MSVC) /* Emulate gnu abi on an msvc compiler */ typedef zig_f128 zig_c_longdouble; #else /* Target and compiler abi match */ typedef long double zig_c_longdouble; #endif #define zig_bitCast_float(Type, ReprType) \ static inline zig_##Type zig_bitCast_##Type(ReprType repr) { \ zig_##Type result; \ memcpy(&result, &repr, sizeof(result)); \ return result; \ } zig_bitCast_float(f16, uint16_t) zig_bitCast_float(f32, uint32_t) zig_bitCast_float(f64, uint64_t) zig_bitCast_float(f80, zig_u128) zig_bitCast_float(f128, zig_u128) #define zig_convert_builtin(ExternResType, ResType, operation, ExternArgType, ArgType, version) \ zig_extern ExternResType zig_expand_concat(zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType), version)(ExternArgType); \ static inline ResType zig_expand_concat(zig_expand_concat(zig_##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType)(ArgType arg) { \ ResType res; \ ExternResType extern_res; \ ExternArgType extern_arg; \ memcpy(&extern_arg, &arg, sizeof(extern_arg)); \ extern_res = zig_expand_concat(zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType), version)(extern_arg); \ memcpy(&res, &extern_res, sizeof(res)); \ return extern_res; \ } zig_convert_builtin(zig_compiler_rt_f16, zig_f16, trunc, zig_f32, zig_f32, 2) zig_convert_builtin(zig_compiler_rt_f16, zig_f16, trunc, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f16, zig_f16, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f16, zig_f16, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f32, zig_f32, extend, zig_compiler_rt_f16, zig_f16, 2) zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f64, zig_f64, extend, zig_compiler_rt_f16, zig_f16, 2) zig_convert_builtin(zig_f64, zig_f64, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f64, zig_f64, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f16, zig_f16, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f32, zig_f32, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f80, zig_f80, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f16, zig_f16, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f32, zig_f32, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f80, zig_f80, 2) #ifdef __ARM_EABI__ zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_d2f(zig_f64); static inline zig_f32 zig_truncdfsf(zig_f64 arg) { return __aeabi_d2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_f2d(zig_f32); static inline zig_f64 zig_extendsfdf(zig_f32 arg) { return __aeabi_f2d(arg); } #else /* __ARM_EABI__ */ zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f64, zig_f64, extend, zig_f32, zig_f32, 2) #endif /* __ARM_EABI__ */ #define zig_float_negate_builtin_0(w, c, sb) \ zig_expand_concat(zig_xor_, zig_repr_f##w)(arg, zig_make_f##w(-0x0.0p0, c sb)) #define zig_float_negate_builtin_1(w, c, sb) -arg #define zig_float_negate_builtin(w, c, sb) \ static inline zig_f##w zig_neg_f##w(zig_f##w arg) { \ return zig_expand_concat(zig_float_negate_builtin_, zig_has_f##w)(w, c, sb); \ } zig_float_negate_builtin(16, , UINT16_C(1) << 15 ) zig_float_negate_builtin(32, , UINT32_C(1) << 31 ) zig_float_negate_builtin(64, , UINT64_C(1) << 63 ) zig_float_negate_builtin(80, zig_make_u128, (UINT64_C(1) << 15, UINT64_C(0))) zig_float_negate_builtin(128, zig_make_u128, (UINT64_C(1) << 63, UINT64_C(0))) #define zig_float_less_builtin_0(Type, operation) \ zig_extern int32_t zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_zig_##Type), 2)(zig_##Type, zig_##Type); \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return zig_expand_concat(zig_expand_concat(__##operation, zig_compiler_rt_abbrev_zig_##Type), 2)(lhs, rhs); \ } #define zig_float_less_builtin_1(Type, operation) \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (!(lhs <= rhs) - (lhs < rhs)); \ } #define zig_float_greater_builtin_0(Type, operation) \ zig_float_less_builtin_0(Type, operation) #define zig_float_greater_builtin_1(Type, operation) \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return ((lhs > rhs) - !(lhs >= rhs)); \ } #define zig_float_binary_builtin_0(Type, operation, operator) \ zig_extern zig_##Type zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_zig_##Type), 3)(zig_##Type, zig_##Type); \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return zig_expand_concat(zig_expand_concat(__##operation, zig_compiler_rt_abbrev_zig_##Type), 3)(lhs, rhs); \ } #define zig_float_binary_builtin_1(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return lhs operator rhs; \ } #define zig_common_float_builtins(w) \ zig_convert_builtin( int64_t, int64_t, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_i128, zig_i128, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_u128, zig_u128, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, int64_t, int64_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, zig_i128, zig_i128, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, zig_u128, zig_u128, ) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, cmp) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, ne) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, eq) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, lt) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, le) \ zig_expand_concat(zig_float_greater_builtin_, zig_has_f##w)(f##w, gt) \ zig_expand_concat(zig_float_greater_builtin_, zig_has_f##w)(f##w, ge) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, add, +) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, sub, -) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, mul, *) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, div, /) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(sqrt)))(zig_f##w, zig_sqrt_f##w, zig_libc_name_f##w(sqrt), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(sin)))(zig_f##w, zig_sin_f##w, zig_libc_name_f##w(sin), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(cos)))(zig_f##w, zig_cos_f##w, zig_libc_name_f##w(cos), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(tan)))(zig_f##w, zig_tan_f##w, zig_libc_name_f##w(tan), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(exp)))(zig_f##w, zig_exp_f##w, zig_libc_name_f##w(exp), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(exp2)))(zig_f##w, zig_exp2_f##w, zig_libc_name_f##w(exp2), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log)))(zig_f##w, zig_log_f##w, zig_libc_name_f##w(log), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log2)))(zig_f##w, zig_log2_f##w, zig_libc_name_f##w(log2), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log10)))(zig_f##w, zig_log10_f##w, zig_libc_name_f##w(log10), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fabs)))(zig_f##w, zig_abs_f##w, zig_libc_name_f##w(fabs), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(floor)))(zig_f##w, zig_floor_f##w, zig_libc_name_f##w(floor), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(ceil)))(zig_f##w, zig_ceil_f##w, zig_libc_name_f##w(ceil), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(round)))(zig_f##w, zig_round_f##w, zig_libc_name_f##w(round), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(trunc)))(zig_f##w, zig_trunc_f##w, zig_libc_name_f##w(trunc), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmod)))(zig_f##w, zig_fmod_f##w, zig_libc_name_f##w(fmod), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmin)))(zig_f##w, zig_min_f##w, zig_libc_name_f##w(fmin), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmax)))(zig_f##w, zig_max_f##w, zig_libc_name_f##w(fmax), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fma)))(zig_f##w, zig_fma_f##w, zig_libc_name_f##w(fma), (zig_f##w x, zig_f##w y, zig_f##w z), (x, y, z)) \ \ static inline zig_f##w zig_div_trunc_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_trunc_f##w(zig_div_f##w(lhs, rhs)); \ } \ \ static inline zig_f##w zig_div_floor_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_floor_f##w(zig_div_f##w(lhs, rhs)); \ } \ \ static inline zig_f##w zig_mod_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_sub_f##w(lhs, zig_mul_f##w(zig_div_floor_f##w(lhs, rhs), rhs)); \ } zig_common_float_builtins(16) zig_common_float_builtins(32) zig_common_float_builtins(64) zig_common_float_builtins(80) zig_common_float_builtins(128) #define zig_float_builtins(w) \ zig_convert_builtin( int32_t, int32_t, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(uint32_t, uint32_t, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(uint64_t, uint64_t, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, int32_t, int32_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, uint32_t, uint32_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, uint64_t, uint64_t, ) zig_float_builtins(16) zig_float_builtins(80) zig_float_builtins(128) #ifdef __ARM_EABI__ zig_extern zig_callconv(pcs("aapcs")) int32_t __aeabi_f2iz(zig_f32); static inline int32_t zig_fixsfsi(zig_f32 arg) { return __aeabi_f2iz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint32_t __aeabi_f2uiz(zig_f32); static inline uint32_t zig_fixunssfsi(zig_f32 arg) { return __aeabi_f2uiz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint64_t __aeabi_f2ulz(zig_f32); static inline uint64_t zig_fixunssfdi(zig_f32 arg) { return __aeabi_f2ulz(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_i2f(int32_t); static inline zig_f32 zig_floatsisf(int32_t arg) { return __aeabi_i2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_ui2f(uint32_t); static inline zig_f32 zig_floatunsisf(uint32_t arg) { return __aeabi_ui2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_ul2f(uint64_t); static inline zig_f32 zig_floatundisf(uint64_t arg) { return __aeabi_ul2f(arg); } zig_extern zig_callconv(pcs("aapcs")) int32_t __aeabi_d2iz(zig_f64); static inline int32_t zig_fixdfsi(zig_f64 arg) { return __aeabi_d2iz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint32_t __aeabi_d2uiz(zig_f64); static inline uint32_t zig_fixunsdfsi(zig_f64 arg) { return __aeabi_d2uiz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint64_t __aeabi_d2ulz(zig_f64); static inline uint64_t zig_fixunsdfdi(zig_f64 arg) { return __aeabi_d2ulz(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_i2d(int32_t); static inline zig_f64 zig_floatsidf(int32_t arg) { return __aeabi_i2d(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_ui2d(uint32_t); static inline zig_f64 zig_floatunsidf(uint32_t arg) { return __aeabi_ui2d(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_ul2d(uint64_t); static inline zig_f64 zig_floatundidf(uint64_t arg) { return __aeabi_ul2d(arg); } #else /* __ARM_EABI__ */ zig_float_builtins(32) zig_float_builtins(64) #endif /* __ARM_EABI__ */ /* ============================ Atomics Support ============================= */ /* Note that atomics should be implemented as macros because most compilers silently discard runtime atomic order information. */ /* Define fallback implementations first that can later be undef'd on compilers with builtin support. */ /* Note that zig_atomicrmw_expected is needed to handle aliasing between res and arg. */ #define zig_atomicrmw_xchg_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_min_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_max_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_xchg_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_add_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_sub_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_and_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_and_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_nand_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_not_##Type(zig_and_##Type(zig_atomicrmw_expected, arg), 128); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_or_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_or_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_xor_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_xor_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_min_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_min_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_max_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_max_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #if (__STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)) || (zig_has_include() && !defined(zig_msvc)) #define zig_c11_atomics #endif #if defined(zig_c11_atomics) #include typedef enum memory_order zig_memory_order; #define zig_memory_order_relaxed memory_order_relaxed #define zig_memory_order_acquire memory_order_acquire #define zig_memory_order_release memory_order_release #define zig_memory_order_acq_rel memory_order_acq_rel #define zig_memory_order_seq_cst memory_order_seq_cst #define zig_atomic(Type) _Atomic(Type) #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) #define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order) #define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order) #undef zig_atomicrmw_xchg_float #define zig_atomicrmw_xchg_float zig_atomicrmw_xchg #undef zig_atomicrmw_add_float #define zig_atomicrmw_add_float zig_atomicrmw_add #undef zig_atomicrmw_sub_float #define zig_atomicrmw_sub_float zig_atomicrmw_sub #elif defined(zig_gnuc) typedef int zig_memory_order; #define zig_memory_order_relaxed __ATOMIC_RELAXED #define zig_memory_order_acquire __ATOMIC_ACQUIRE #define zig_memory_order_release __ATOMIC_RELEASE #define zig_memory_order_acq_rel __ATOMIC_ACQ_REL #define zig_memory_order_seq_cst __ATOMIC_SEQ_CST #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, (ReprType *)&(expected), (ReprType *)&(desired), false, succ, fail) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, (ReprType *)&(expected), (ReprType *)&(desired), true, succ, fail) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, (ReprType *)&(arg), &(res), order) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) #define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, (ReprType *)&(arg), order) #define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order) #undef zig_atomicrmw_xchg_float #define zig_atomicrmw_xchg_float zig_atomicrmw_xchg #elif defined(zig_msvc) && defined(zig_x86) #define zig_memory_order_relaxed 0 #define zig_memory_order_acquire 2 #define zig_memory_order_release 3 #define zig_memory_order_acq_rel 4 #define zig_memory_order_seq_cst 5 #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_msvc_cmpxchg_##Type(obj, &(expected), desired) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(obj, expected, desired, succ, fail, Type, ReprType) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg) #define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg) #define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##order##_##Type(obj) /* TODO: zig_msvc && (zig_thumb || zig_aarch64) */ #else #define zig_memory_order_relaxed 0 #define zig_memory_order_acquire 2 #define zig_memory_order_release 3 #define zig_memory_order_acq_rel 4 #define zig_memory_order_seq_cst 5 #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable #endif #if !defined(zig_c11_atomics) && defined(zig_msvc) && defined(zig_x86) /* TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64 */ #define zig_msvc_atomics(ZigType, Type, SigType, suffix, iso_suffix) \ static inline bool zig_msvc_cmpxchg_##ZigType(Type volatile* obj, Type* expected, Type desired) { \ Type comparand = *expected; \ Type initial = _InterlockedCompareExchange##suffix((SigType volatile*)obj, (SigType)desired, (SigType)comparand); \ bool exchanged = initial == comparand; \ if (!exchanged) { \ *expected = initial; \ } \ return exchanged; \ } \ static inline Type zig_msvc_atomicrmw_xchg_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedExchange##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_add_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedExchangeAdd##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_sub_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = prev - value; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_or_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedOr##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_xor_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedXor##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_and_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedAnd##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_nand_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = ~(prev & value); \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_min_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = value < prev ? value : prev; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_max_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = value > prev ? value : prev; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline void zig_msvc_atomic_store_##ZigType(Type volatile* obj, Type value) { \ (void)_InterlockedExchange##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_relaxed_##ZigType(Type volatile* obj) { \ return __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_acquire_##ZigType(Type volatile* obj) { \ Type val = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ return val; \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_seq_cst_##ZigType(Type volatile* obj) { \ Type val = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ return val; \ } zig_msvc_atomics( u8, uint8_t, char, 8, 8) zig_msvc_atomics( i8, int8_t, char, 8, 8) zig_msvc_atomics(u16, uint16_t, short, 16, 16) zig_msvc_atomics(i16, int16_t, short, 16, 16) zig_msvc_atomics(u32, uint32_t, long, , 32) zig_msvc_atomics(i32, int32_t, long, , 32) #if defined(zig_x86_64) zig_msvc_atomics(u64, uint64_t, __int64, 64, 64) zig_msvc_atomics(i64, int64_t, __int64, 64, 64) #endif #define zig_msvc_flt_atomics(Type, SigType, suffix, iso_suffix) \ static inline bool zig_msvc_cmpxchg_##Type(zig_##Type volatile* obj, zig_##Type* expected, zig_##Type desired) { \ SigType exchange; \ SigType comparand; \ SigType initial; \ bool success; \ memcpy(&comparand, expected, sizeof(comparand)); \ memcpy(&exchange, &desired, sizeof(exchange)); \ initial = _InterlockedCompareExchange##suffix((SigType volatile*)obj, exchange, comparand); \ success = initial == comparand; \ if (!success) memcpy(expected, &initial, sizeof(*expected)); \ return success; \ } \ static inline void zig_msvc_atomic_store_##Type(zig_##Type volatile* obj, zig_##Type arg) { \ SigType value; \ memcpy(&value, &arg, sizeof(value)); \ (void)_InterlockedExchange##suffix((SigType volatile*)obj, value); \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_relaxed_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_acquire_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_seq_cst_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } zig_msvc_flt_atomics(f32, long, , 32) #if defined(zig_x86_64) zig_msvc_flt_atomics(f64, int64_t, 64, 64) #endif #if defined(zig_x86_32) static inline void zig_msvc_atomic_barrier() { int32_t barrier; __asm { xchg barrier, eax } } static inline void* zig_msvc_atomicrmw_xchg_p32(void volatile* obj, void* arg) { return _InterlockedExchangePointer(obj, arg); } static inline void zig_msvc_atomic_store_p32(void volatile* obj, void* arg) { (void)_InterlockedExchangePointer(obj, arg); } static inline void* zig_msvc_atomic_load_zig_memory_order_relaxed_p32(void volatile* obj) { return (void*)__iso_volatile_load32(obj); } static inline void* zig_msvc_atomic_load_zig_memory_order_acquire_p32(void volatile* obj) { void* val = (void*)__iso_volatile_load32(obj); _ReadWriteBarrier(); return val; } static inline void* zig_msvc_atomic_load_zig_memory_order_seq_cst_p32(void volatile* obj) { return zig_msvc_atomic_load_zig_memory_order_acquire_p32(obj); } static inline bool zig_msvc_cmpxchg_p32(void volatile* obj, void* expected, void* desired) { void* comparand = *(void**)expected; void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand); bool success = initial == comparand; if (!success) *(void**)expected = initial; return success; } #else /* zig_x86_32 */ static inline void* zig_msvc_atomicrmw_xchg_p64(void volatile* obj, void* arg) { return _InterlockedExchangePointer(obj, arg); } static inline void zig_msvc_atomic_store_p64(void volatile* obj, void* arg) { (void)_InterlockedExchangePointer(obj, arg); } static inline void* zig_msvc_atomic_load_zig_memory_order_relaxed_p64(void volatile* obj) { return (void*)__iso_volatile_load64(obj); } static inline void* zig_msvc_atomic_load_zig_memory_order_acquire_p64(void volatile* obj) { void* val = (void*)__iso_volatile_load64(obj); _ReadWriteBarrier(); return val; } static inline void* zig_msvc_atomic_load_zig_memory_order_seq_cst_p64(void volatile* obj) { return zig_msvc_atomic_load_zig_memory_order_acquire_p64(obj); } static inline bool zig_msvc_cmpxchg_p64(void volatile* obj, void* expected, void* desired) { void* comparand = *(void**)expected; void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand); bool success = initial == comparand; if (!success) *(void**)expected = initial; return success; } static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expected, zig_u128 desired) { return _InterlockedCompareExchange128((__int64 volatile*)obj, (__int64)zig_hi_u128(desired), (__int64)zig_lo_u128(desired), (__int64*)expected); } static inline zig_u128 zig_msvc_atomic_load_u128(zig_u128 volatile* obj) { zig_u128 expected = zig_make_u128(UINT64_C(0), UINT64_C(0)); (void)zig_cmpxchg_strong(obj, expected, expected, zig_memory_order_seq_cst, zig_memory_order_seq_cst, u128, zig_u128); return expected; } static inline void zig_msvc_atomic_store_u128(zig_u128 volatile* obj, zig_u128 arg) { zig_u128 expected = zig_make_u128(UINT64_C(0), UINT64_C(0)); while (!zig_cmpxchg_weak(obj, expected, arg, zig_memory_order_seq_cst, zig_memory_order_seq_cst, u128, zig_u128)); } static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) { return _InterlockedCompareExchange128((__int64 volatile*)obj, (__int64)zig_hi_i128(desired), (__int64)zig_lo_i128(desired), (__int64*)expected); } static inline zig_i128 zig_msvc_atomic_load_i128(zig_i128 volatile* obj) { zig_i128 expected = zig_make_i128(INT64_C(0), UINT64_C(0)); (void)zig_cmpxchg_strong(obj, expected, expected, zig_memory_order_seq_cst, zig_memory_order_seq_cst, i128, zig_i128); return expected; } static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 arg) { zig_i128 expected = zig_make_i128(INT64_C(0), UINT64_C(0)); while (!zig_cmpxchg_weak(obj, expected, arg, zig_memory_order_seq_cst, zig_memory_order_seq_cst, i128, zig_i128)); } #endif /* zig_x86_32 */ #endif /* !zig_c11_atomics && zig_msvc && zig_x86 */ /* ======================== Special Case Intrinsics ========================= */ #if defined(zig_msvc) #include #endif #if defined(zig_thumb) static inline void* zig_thumb_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2); #elif defined(zig_gnuc_asm) __asm__ ("mrc p15, 0, %[ptr], c13, c0, 2" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_aarch64) static inline void* zig_aarch64_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readx18qword(0x0); #elif defined(zig_gnuc_asm) __asm__ ("mov %[ptr], x18" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_x86_32) static inline void* zig_x86_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readfsdword(0x18); #elif defined(zig_gnuc_asm) __asm__ ("movl %%fs:0x18, %[ptr]" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_x86_64) static inline void* zig_x86_64_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readgsqword(0x30); #elif defined(zig_gnuc_asm) __asm__ ("movq %%gs:0x30, %[ptr]" : [ptr] "=r" (teb)); #endif return teb; } #endif #if defined(zig_x86) static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) { #if defined(zig_msvc) int cpu_info[4]; __cpuidex(cpu_info, leaf_id, subid); *eax = (uint32_t)cpu_info[0]; *ebx = (uint32_t)cpu_info[1]; *ecx = (uint32_t)cpu_info[2]; *edx = (uint32_t)cpu_info[3]; #elif defined(zig_gnuc_asm) __asm__("cpuid" : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) : "a"(leaf_id), "c"(subid)); #else *eax = 0; *ebx = 0; *ecx = 0; *edx = 0; #endif } static inline uint32_t zig_x86_get_xcr0(void) { #if defined(zig_msvc) return (uint32_t)_xgetbv(0); #elif defined(zig_gnuc_asm) uint32_t eax; uint32_t edx; __asm__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); return eax; #else *eax = 0; *ebx = 0; *ecx = 0; *edx = 0; #endif } #endif codspeed-4.4.1/instrument-hooks/flake.lock000064400000000000000000000027261046102023000167140ustar 00000000000000{ "nodes": { "flake-utils": { "inputs": { "systems": "systems" }, "locked": { "lastModified": 1731533236, "narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=", "owner": "numtide", "repo": "flake-utils", "rev": "11707dc2f618dd54ca8739b309ec4fc024de578b", "type": "github" }, "original": { "owner": "numtide", "repo": "flake-utils", "type": "github" } }, "nixpkgs": { "locked": { "lastModified": 1752866191, "narHash": "sha256-NV4S2Lf2hYmZQ3Qf4t/YyyBaJNuxLPyjzvDma0zPp/M=", "owner": "NixOS", "repo": "nixpkgs", "rev": "f01fe91b0108a7aff99c99f2e9abbc45db0adc2a", "type": "github" }, "original": { "owner": "NixOS", "ref": "nixos-25.05", "repo": "nixpkgs", "type": "github" } }, "root": { "inputs": { "flake-utils": "flake-utils", "nixpkgs": "nixpkgs" } }, "systems": { "locked": { "lastModified": 1681028828, "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", "owner": "nix-systems", "repo": "default", "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", "type": "github" }, "original": { "owner": "nix-systems", "repo": "default", "type": "github" } } }, "root": "root", "version": 7 } codspeed-4.4.1/instrument-hooks/flake.nix000064400000000000000000000021451046102023000165550ustar 00000000000000{ description = "CodSpeed instrument hooks development environment"; inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-25.05"; flake-utils.url = "github:numtide/flake-utils"; }; outputs = { self, nixpkgs, flake-utils, }: flake-utils.lib.eachDefaultSystem ( system: let pkgs = import nixpkgs { inherit system; }; commonBuildInputs = with pkgs; [ just zig clang cmake bazelisk python3 ]; in { devShells = { default = pkgs.mkShell { buildInputs = commonBuildInputs; shellHook = '' echo "Instrument hooks development environment" ''; }; lsp = pkgs.mkShell { buildInputs = with pkgs; [ zls clang-tools ] ++ commonBuildInputs; shellHook = '' echo "Instrument hooks development environment with LSP" ''; }; }; } ); } codspeed-4.4.1/instrument-hooks/includes/callgrind.h000064400000000000000000000130471046102023000206740ustar 00000000000000/* ---------------------------------------------------------------- Notice that the following BSD-style license applies to this one file (callgrind.h) only. The rest of Valgrind is licensed under the terms of the GNU General Public License, version 2, unless otherwise indicated. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- This file is part of callgrind, a valgrind tool for cache simulation and call tree tracing. Copyright (C) 2003-2017 Josef Weidendorfer. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------- Notice that the above BSD-style license applies to this one file (callgrind.h) only. The entire rest of Valgrind is licensed under the terms of the GNU General Public License, version 2. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- */ #ifndef __CALLGRIND_H #define __CALLGRIND_H #include "valgrind.h" /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! This enum comprises an ABI exported by Valgrind to programs which use client requests. DO NOT CHANGE THE ORDER OF THESE ENTRIES, NOR DELETE ANY -- add new ones at the end. The identification ('C','T') for Callgrind has historical reasons: it was called "Calltree" before. Besides, ('C','G') would clash with cachegrind. */ typedef enum { VG_USERREQ__DUMP_STATS = VG_USERREQ_TOOL_BASE('C', 'T'), VG_USERREQ__ZERO_STATS, VG_USERREQ__TOGGLE_COLLECT, VG_USERREQ__DUMP_STATS_AT, VG_USERREQ__START_INSTRUMENTATION, VG_USERREQ__STOP_INSTRUMENTATION } Vg_CallgrindClientRequest; /* Dump current state of cost centers, and zero them afterwards */ #define CALLGRIND_DUMP_STATS \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS, 0, 0, 0, 0, 0) /* Dump current state of cost centers, and zero them afterwards. The argument is appended to a string stating the reason which triggered the dump. This string is written as a description field into the profile data dump. */ #define CALLGRIND_DUMP_STATS_AT(pos_str) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DUMP_STATS_AT, pos_str, 0, 0, 0, \ 0) /* Zero cost centers */ #define CALLGRIND_ZERO_STATS \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__ZERO_STATS, 0, 0, 0, 0, 0) /* Toggles collection state. The collection state specifies whether the happening of events should be noted or if they are to be ignored. Events are noted by increment of counters in a cost center */ #define CALLGRIND_TOGGLE_COLLECT \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__TOGGLE_COLLECT, 0, 0, 0, 0, 0) /* Start full callgrind instrumentation if not already switched on. When cache simulation is done, it will flush the simulated cache; this will lead to an artificial cache warmup phase afterwards with cache misses which would not have happened in reality. */ #define CALLGRIND_START_INSTRUMENTATION \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__START_INSTRUMENTATION, 0, 0, 0, \ 0, 0) /* Stop full callgrind instrumentation if not already switched off. This flushes Valgrinds translation cache, and does no additional instrumentation afterwards, which effectivly will run at the same speed as the "none" tool (ie. at minimal slowdown). Use this to bypass Callgrind aggregation for uninteresting code parts. To start Callgrind in this mode to ignore the setup phase, use the option "--instr-atstart=no". */ #define CALLGRIND_STOP_INSTRUMENTATION \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STOP_INSTRUMENTATION, 0, 0, 0, \ 0, 0) #endif /* __CALLGRIND_H */ codspeed-4.4.1/instrument-hooks/includes/compat.h000064400000000000000000000020201046102023000202050ustar 00000000000000// Compatibility shims for cross-platform compilation // This header provides compatibility macros for different libc implementations #ifndef COMPAT_H #define COMPAT_H // Detect musl libc // Musl doesn't define a specific macro, but we can detect it by checking for // the absence of __GLIBC__ on Linux systems, or by checking features.h #if defined(__linux__) && !defined(__GLIBC__) && !defined(__BIONIC__) #define COMPAT_MUSL 1 #endif // Musl libc compatibility // Musl doesn't have separate *64 functions because all functions are 64-bit // clean. Map the glibc LFS64 (Large File Support) functions to their standard // equivalents. #if defined(COMPAT_MUSL) #define openat64 openat #define fopen64 fopen #define freopen64 freopen #define lseek64 lseek #define fseeko64 fseeko #define ftello64 ftello #define fgetpos64 fgetpos #define fsetpos64 fsetpos #define stat64 stat #define fstat64 fstat #define lstat64 lstat #define fstatat64 fstatat #define mmap64 mmap #define off64_t off_t #define ino64_t ino_t #endif #endif // COMPAT_H codspeed-4.4.1/instrument-hooks/includes/core.h000064400000000000000000000051401046102023000176600ustar 00000000000000// This file was manually created and exposes the functions of this library. // TODO: Can we automatically generate this file? #ifndef CORE_H #define CORE_H #include #include #include #ifdef __cplusplus extern "C" { #endif #ifndef _WIN32 #include "callgrind.h" #include "valgrind.h" #else #define CALLGRIND_START_INSTRUMENTATION #define CALLGRIND_STOP_INSTRUMENTATION #define CALLGRIND_ZERO_STATS #endif typedef uint64_t *InstrumentHooks; InstrumentHooks *instrument_hooks_init(void); void instrument_hooks_deinit(InstrumentHooks *); bool instrument_hooks_is_instrumented(InstrumentHooks *); uint8_t instrument_hooks_start_benchmark(InstrumentHooks *); uint8_t instrument_hooks_stop_benchmark(InstrumentHooks *); uint8_t instrument_hooks_set_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); // Deprecated: use instrument_hooks_set_executed_benchmark instead uint8_t instrument_hooks_executed_benchmark(InstrumentHooks *, int32_t pid, const char *uri); uint8_t instrument_hooks_set_integration(InstrumentHooks *, const char *name, const char *version); #define MARKER_TYPE_SAMPLE_START 0 #define MARKER_TYPE_SAMPLE_END 1 #define MARKER_TYPE_BENCHMARK_START 2 #define MARKER_TYPE_BENCHMARK_END 3 uint8_t instrument_hooks_add_marker(InstrumentHooks *, uint32_t pid, uint8_t marker_type, uint64_t timestamp); uint64_t instrument_hooks_current_timestamp(void); void callgrind_start_instrumentation(); void callgrind_stop_instrumentation(); // Feature flags for instrument hooks typedef enum { FEATURE_DISABLE_CALLGRIND_MARKERS = 0, } instrument_hooks_feature_t; void instrument_hooks_set_feature(instrument_hooks_feature_t feature, bool enabled); // Header functions that will be inlined. This can be used by languages that // directly consume the headers such as C or C++. This will allow for more // precise tracking of the benchmark performance. static inline uint8_t instrument_hooks_start_benchmark_inline( InstrumentHooks *instance) { instrument_hooks_set_feature(FEATURE_DISABLE_CALLGRIND_MARKERS, true); if (instrument_hooks_start_benchmark(instance) != 0) { return 1; } CALLGRIND_ZERO_STATS; CALLGRIND_START_INSTRUMENTATION; return 0; } static inline uint8_t instrument_hooks_stop_benchmark_inline( InstrumentHooks *instance) { CALLGRIND_STOP_INSTRUMENTATION; return instrument_hooks_stop_benchmark(instance); } #ifdef __cplusplus } #endif #endif codspeed-4.4.1/instrument-hooks/includes/valgrind.h000064400000000000000000014725571046102023000205630ustar 00000000000000/* -*- c -*- ---------------------------------------------------------------- Notice that the following BSD-style license applies to this one file (valgrind.h) only. The rest of Valgrind is licensed under the terms of the GNU General Public License, version 2, unless otherwise indicated. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- This file is part of Valgrind, a dynamic binary instrumentation framework. Copyright (C) 2000-2017 Julian Seward. All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. 2. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment in the product documentation would be appreciated but is not required. 3. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software. 4. The name of the author may not be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ---------------------------------------------------------------- Notice that the above BSD-style license applies to this one file (valgrind.h) only. The entire rest of Valgrind is licensed under the terms of the GNU General Public License, version 2. See the COPYING file in the source distribution for details. ---------------------------------------------------------------- */ /* This file is for inclusion into client (your!) code. You can use these macros to manipulate and query Valgrind's execution inside your own programs. The resulting executables will still run without Valgrind, just a little bit more slowly than they otherwise would, but otherwise unchanged. When not running on valgrind, each client request consumes very few (eg. 7) instructions, so the resulting performance loss is negligible unless you plan to execute client requests millions of times per second. Nevertheless, if that is still a problem, you can compile with the NVALGRIND symbol defined (gcc -DNVALGRIND) so that client requests are not even compiled in. */ #ifndef __VALGRIND_H #define __VALGRIND_H /* ------------------------------------------------------------------ */ /* VERSION NUMBER OF VALGRIND */ /* ------------------------------------------------------------------ */ /* Specify Valgrind's version number, so that user code can conditionally compile based on our version number. Note that these were introduced at version 3.6 and so do not exist in version 3.5 or earlier. The recommended way to use them to check for "version X.Y or later" is (eg) #if defined(__VALGRIND_MAJOR__) && defined(__VALGRIND_MINOR__) \ && (__VALGRIND_MAJOR__ > 3 \ || (__VALGRIND_MAJOR__ == 3 && __VALGRIND_MINOR__ >= 6)) */ #define __VALGRIND_MAJOR__ 3 #define __VALGRIND_MINOR__ 24 #include /* Nb: this file might be included in a file compiled with -ansi. So we can't use C++ style "//" comments nor the "asm" keyword (instead use "__asm__"). */ /* Derive some tags indicating what the target platform is. Note that in this file we're using the compiler's CPP symbols for identifying architectures, which are different to the ones we use within the rest of Valgrind. Note, __powerpc__ is active for both 32 and 64-bit PPC, whereas __powerpc64__ is only active for the latter (on Linux, that is). Misc note: how to find out what's predefined in gcc by default: gcc -Wp,-dM somefile.c */ #undef PLAT_x86_darwin #undef PLAT_amd64_darwin #undef PLAT_x86_freebsd #undef PLAT_amd64_freebsd #undef PLAT_arm64_freebsd #undef PLAT_x86_win32 #undef PLAT_amd64_win64 #undef PLAT_x86_linux #undef PLAT_amd64_linux #undef PLAT_ppc32_linux #undef PLAT_ppc64be_linux #undef PLAT_ppc64le_linux #undef PLAT_arm_linux #undef PLAT_arm64_linux #undef PLAT_s390x_linux #undef PLAT_mips32_linux #undef PLAT_mips64_linux #undef PLAT_nanomips_linux #undef PLAT_x86_solaris #undef PLAT_amd64_solaris #if defined(__APPLE__) && defined(__i386__) # define PLAT_x86_darwin 1 #elif defined(__APPLE__) && defined(__x86_64__) # define PLAT_amd64_darwin 1 #elif defined(__FreeBSD__) && defined(__i386__) # define PLAT_x86_freebsd 1 #elif defined(__FreeBSD__) && defined(__amd64__) # define PLAT_amd64_freebsd 1 #elif defined(__FreeBSD__) && defined(__aarch64__) && !defined(__arm__) # define PLAT_arm64_freebsd 1 #elif (defined(__MINGW32__) && defined(__i386__)) \ || defined(__CYGWIN32__) \ || (defined(_WIN32) && defined(_M_IX86)) # define PLAT_x86_win32 1 #elif (defined(__MINGW32__) && defined(__x86_64__)) \ || (defined(_WIN32) && defined(_M_X64)) /* __MINGW32__ and _WIN32 are defined in 64 bit mode as well. */ # define PLAT_amd64_win64 1 #elif defined(__linux__) && defined(__i386__) # define PLAT_x86_linux 1 #elif defined(__linux__) && defined(__x86_64__) && !defined(__ILP32__) # define PLAT_amd64_linux 1 #elif defined(__linux__) && defined(__powerpc__) && !defined(__powerpc64__) # define PLAT_ppc32_linux 1 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF != 2 /* Big Endian uses ELF version 1 */ # define PLAT_ppc64be_linux 1 #elif defined(__linux__) && defined(__powerpc__) && defined(__powerpc64__) && _CALL_ELF == 2 /* Little Endian uses ELF version 2 */ # define PLAT_ppc64le_linux 1 #elif defined(__linux__) && defined(__arm__) && !defined(__aarch64__) # define PLAT_arm_linux 1 #elif defined(__linux__) && defined(__aarch64__) && !defined(__arm__) # define PLAT_arm64_linux 1 #elif defined(__linux__) && defined(__s390__) && defined(__s390x__) # define PLAT_s390x_linux 1 #elif defined(__linux__) && defined(__mips__) && (__mips==64) # define PLAT_mips64_linux 1 #elif defined(__linux__) && defined(__mips__) && (__mips==32) # define PLAT_mips32_linux 1 #elif defined(__linux__) && defined(__nanomips__) # define PLAT_nanomips_linux 1 #elif defined(__sun) && defined(__i386__) # define PLAT_x86_solaris 1 #elif defined(__sun) && defined(__x86_64__) # define PLAT_amd64_solaris 1 #else /* If we're not compiling for our target platform, don't generate any inline asms. */ # if !defined(NVALGRIND) # define NVALGRIND 1 # endif #endif /* ------------------------------------------------------------------ */ /* ARCHITECTURE SPECIFICS for SPECIAL INSTRUCTIONS. There is nothing */ /* in here of use to end-users -- skip to the next section. */ /* ------------------------------------------------------------------ */ /* * VALGRIND_DO_CLIENT_REQUEST(): a statement that invokes a Valgrind client * request. Accepts both pointers and integers as arguments. * * VALGRIND_DO_CLIENT_REQUEST_STMT(): a statement that invokes a Valgrind * client request that does not return a value. * VALGRIND_DO_CLIENT_REQUEST_EXPR(): a C expression that invokes a Valgrind * client request and whose value equals the client request result. Accepts * both pointers and integers as arguments. Note that such calls are not * necessarily pure functions -- they may have side effects. */ #define VALGRIND_DO_CLIENT_REQUEST(_zzq_rlval, _zzq_default, \ _zzq_request, _zzq_arg1, _zzq_arg2, \ _zzq_arg3, _zzq_arg4, _zzq_arg5) \ do { (_zzq_rlval) = VALGRIND_DO_CLIENT_REQUEST_EXPR((_zzq_default), \ (_zzq_request), (_zzq_arg1), (_zzq_arg2), \ (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0) #define VALGRIND_DO_CLIENT_REQUEST_STMT(_zzq_request, _zzq_arg1, \ _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ do { (void) VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ (_zzq_request), (_zzq_arg1), (_zzq_arg2), \ (_zzq_arg3), (_zzq_arg4), (_zzq_arg5)); } while (0) #if defined(NVALGRIND) /* Define NVALGRIND to completely remove the Valgrind magic sequence from the compiled code (analogous to NDEBUG's effects on assert()) */ #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ (_zzq_default) #else /* ! NVALGRIND */ /* The following defines the magic code sequences which the JITter spots and handles magically. Don't look too closely at them as they will rot your brain. The assembly code sequences for all architectures is in this one file. This is because this file must be stand-alone, and we don't want to have multiple files. For VALGRIND_DO_CLIENT_REQUEST, we must ensure that the default value gets put in the return slot, so that everything works when this is executed not under Valgrind. Args are passed in a memory block, and so there's no intrinsic limit to the number that could be passed, but it's currently five. The macro args are: _zzq_rlval result lvalue _zzq_default default value (result returned when running on real CPU) _zzq_request request code _zzq_arg1..5 request params The other two macros are used to support function wrapping, and are a lot simpler. VALGRIND_GET_NR_CONTEXT returns the value of the guest's NRADDR pseudo-register and whatever other information is needed to safely run the call original from the wrapper: on ppc64-linux, the R2 value at the divert point is also needed. This information is abstracted into a user-visible type, OrigFn. VALGRIND_CALL_NOREDIR_* behaves the same as the following on the guest, but guarantees that the branch instruction will not be redirected: x86: call *%eax, amd64: call *%rax, ppc32/ppc64: branch-and-link-to-r11. VALGRIND_CALL_NOREDIR is just text, not a complete inline asm, since it needs to be combined with more magic inline asm stuff to be useful. */ /* ----------------- x86-{linux,darwin,solaris} ---------------- */ #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin) \ || (defined(PLAT_x86_win32) && defined(__GNUC__)) \ || defined(PLAT_x86_solaris) || defined(PLAT_x86_freebsd) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "roll $3, %%edi ; roll $13, %%edi\n\t" \ "roll $29, %%edi ; roll $19, %%edi\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %EDX = client_request ( %EAX ) */ \ "xchgl %%ebx,%%ebx" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), "0" (_zzq_default) \ : "cc", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %EAX = guest_NRADDR */ \ "xchgl %%ecx,%%ecx" \ : "=a" (__addr) \ : \ : "cc", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_EAX \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%EAX */ \ "xchgl %%edx,%%edx\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "xchgl %%edi,%%edi\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_x86_linux || PLAT_x86_darwin || (PLAT_x86_win32 && __GNUC__) || PLAT_x86_solaris */ /* ------------------------- x86-Win32 ------------------------- */ #if defined(PLAT_x86_win32) && !defined(__GNUC__) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #if defined(_MSC_VER) #define __SPECIAL_INSTRUCTION_PREAMBLE \ __asm rol edi, 3 __asm rol edi, 13 \ __asm rol edi, 29 __asm rol edi, 19 #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ valgrind_do_client_request_expr((uintptr_t)(_zzq_default), \ (uintptr_t)(_zzq_request), (uintptr_t)(_zzq_arg1), \ (uintptr_t)(_zzq_arg2), (uintptr_t)(_zzq_arg3), \ (uintptr_t)(_zzq_arg4), (uintptr_t)(_zzq_arg5)) static __inline uintptr_t valgrind_do_client_request_expr(uintptr_t _zzq_default, uintptr_t _zzq_request, uintptr_t _zzq_arg1, uintptr_t _zzq_arg2, uintptr_t _zzq_arg3, uintptr_t _zzq_arg4, uintptr_t _zzq_arg5) { volatile uintptr_t _zzq_args[6]; volatile unsigned int _zzq_result; _zzq_args[0] = (uintptr_t)(_zzq_request); _zzq_args[1] = (uintptr_t)(_zzq_arg1); _zzq_args[2] = (uintptr_t)(_zzq_arg2); _zzq_args[3] = (uintptr_t)(_zzq_arg3); _zzq_args[4] = (uintptr_t)(_zzq_arg4); _zzq_args[5] = (uintptr_t)(_zzq_arg5); __asm { __asm lea eax, _zzq_args __asm mov edx, _zzq_default __SPECIAL_INSTRUCTION_PREAMBLE /* %EDX = client_request ( %EAX ) */ __asm xchg ebx,ebx __asm mov _zzq_result, edx } return _zzq_result; } #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm { __SPECIAL_INSTRUCTION_PREAMBLE \ /* %EAX = guest_NRADDR */ \ __asm xchg ecx,ecx \ __asm mov __addr, eax \ } \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_EAX ERROR #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm { __SPECIAL_INSTRUCTION_PREAMBLE \ __asm xchg edi,edi \ } \ } while (0) #else #error Unsupported compiler. #endif #endif /* PLAT_x86_win32 */ /* ----------------- amd64-{linux,darwin,solaris} --------------- */ #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin) \ || defined(PLAT_amd64_solaris) \ || defined(PLAT_amd64_freebsd) \ || (defined(PLAT_amd64_win64) && defined(__GNUC__)) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rolq $3, %%rdi ; rolq $13, %%rdi\n\t" \ "rolq $61, %%rdi ; rolq $51, %%rdi\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %RDX = client_request ( %RAX ) */ \ "xchgq %%rbx,%%rbx" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), "0" (_zzq_default) \ : "cc", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %RAX = guest_NRADDR */ \ "xchgq %%rcx,%%rcx" \ : "=a" (__addr) \ : \ : "cc", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_RAX \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%RAX */ \ "xchgq %%rdx,%%rdx\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "xchgq %%rdi,%%rdi\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */ /* ------------------------- amd64-Win64 ------------------------- */ #if defined(PLAT_amd64_win64) && !defined(__GNUC__) #error Unsupported compiler. #endif /* PLAT_amd64_win64 */ /* ------------------------ ppc32-linux ------------------------ */ #if defined(PLAT_ppc32_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rlwinm 0,0,3,0,31 ; rlwinm 0,0,13,0,31\n\t" \ "rlwinm 0,0,29,0,31 ; rlwinm 0,0,19,0,31\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned int _zzq_args[6]; \ unsigned int _zzq_result; \ unsigned int* _zzq_ptr; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R11 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc32_linux */ /* ------------------------ ppc64-linux ------------------------ */ #if defined(PLAT_ppc64be_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ unsigned long int r2; /* what tocptr do we need? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ "rotldi 0,0,61 ; rotldi 0,0,51\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned long int _zzq_args[6]; \ unsigned long int _zzq_result; \ unsigned long int* _zzq_ptr; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR_GPR2 */ \ "or 4,4,4\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->r2 = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R11 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc64be_linux */ #if defined(PLAT_ppc64le_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ unsigned long int r2; /* what tocptr do we need? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "rotldi 0,0,3 ; rotldi 0,0,13\n\t" \ "rotldi 0,0,61 ; rotldi 0,0,51\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({ unsigned long int _zzq_args[6]; \ unsigned long int _zzq_result; \ unsigned long int* _zzq_ptr; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ _zzq_ptr = _zzq_args; \ __asm__ volatile("mr 3,%1\n\t" /*default*/ \ "mr 4,%2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = client_request ( %R4 ) */ \ "or 1,1,1\n\t" \ "mr %0,3" /*result*/ \ : "=b" (_zzq_result) \ : "b" (_zzq_default), "b" (_zzq_ptr) \ : "cc", "memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR */ \ "or 2,2,2\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %R3 = guest_NRADDR_GPR2 */ \ "or 4,4,4\n\t" \ "mr %0,3" \ : "=b" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->r2 = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R12 */ \ "or 3,3,3\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or 5,5,5\n\t" \ ); \ } while (0) #endif /* PLAT_ppc64le_linux */ /* ------------------------- arm-linux ------------------------- */ #if defined(PLAT_arm_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "mov r12, r12, ror #3 ; mov r12, r12, ror #13 \n\t" \ "mov r12, r12, ror #29 ; mov r12, r12, ror #19 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("mov r3, %1\n\t" /*default*/ \ "mov r4, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* R3 = client_request ( R4 ) */ \ "orr r10, r10, r10\n\t" \ "mov %0, r3" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "cc","memory", "r3", "r4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* R3 = guest_NRADDR */ \ "orr r11, r11, r11\n\t" \ "mov %0, r3" \ : "=r" (__addr) \ : \ : "cc", "memory", "r3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir *%R4 */ \ "orr r12, r12, r12\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "orr r9, r9, r9\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_arm_linux */ /* ------------------------ arm64-{linux,freebsd} ------------------------- */ #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; #define __SPECIAL_INSTRUCTION_PREAMBLE \ "ror x12, x12, #3 ; ror x12, x12, #13 \n\t" \ "ror x12, x12, #51 ; ror x12, x12, #61 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ \ __extension__ \ ({volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile("mov x3, %1\n\t" /*default*/ \ "mov x4, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* X3 = client_request ( X4 ) */ \ "orr x10, x10, x10\n\t" \ "mov %0, x3" /*result*/ \ : "=r" (_zzq_result) \ : "r" ((unsigned long int)(_zzq_default)), \ "r" (&_zzq_args[0]) \ : "cc","memory", "x3", "x4"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* X3 = guest_NRADDR */ \ "orr x11, x11, x11\n\t" \ "mov %0, x3" \ : "=r" (__addr) \ : \ : "cc", "memory", "x3" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* branch-and-link-to-noredir X8 */ \ "orr x12, x12, x12\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "orr x9, x9, x9\n\t" \ : : : "cc", "memory" \ ); \ } while (0) #endif /* PLAT_arm64_linux || PLAT_arm64_freebsd */ /* ------------------------ s390x-linux ------------------------ */ #if defined(PLAT_s390x_linux) typedef struct { unsigned long int nraddr; /* where's the code? */ } OrigFn; /* __SPECIAL_INSTRUCTION_PREAMBLE will be used to identify Valgrind specific * code. This detection is implemented in platform specific toIR.c * (e.g. VEX/priv/guest_s390_decoder.c). */ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "lr 15,15\n\t" \ "lr 1,1\n\t" \ "lr 2,2\n\t" \ "lr 3,3\n\t" #define __CLIENT_REQUEST_CODE "lr 2,2\n\t" #define __GET_NR_CONTEXT_CODE "lr 3,3\n\t" #define __CALL_NO_REDIR_CODE "lr 4,4\n\t" #define __VEX_INJECT_IR_CODE "lr 5,5\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile(/* r2 = args */ \ "lgr 2,%1\n\t" \ /* r3 = default */ \ "lgr 3,%2\n\t" \ __SPECIAL_INSTRUCTION_PREAMBLE \ __CLIENT_REQUEST_CODE \ /* results = r3 */ \ "lgr %0, 3\n\t" \ : "=d" (_zzq_result) \ : "a" (&_zzq_args[0]), \ "0" ((unsigned long int)_zzq_default) \ : "cc", "2", "3", "memory" \ ); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ __GET_NR_CONTEXT_CODE \ "lgr %0, 3\n\t" \ : "=a" (__addr) \ : \ : "cc", "3", "memory" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_R1 \ __SPECIAL_INSTRUCTION_PREAMBLE \ __CALL_NO_REDIR_CODE #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ __VEX_INJECT_IR_CODE); \ } while (0) #endif /* PLAT_s390x_linux */ /* ------------------------- mips32-linux ---------------- */ #if defined(PLAT_mips32_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; /* .word 0x342 * .word 0x742 * .word 0xC2 * .word 0x4C2*/ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "srl $0, $0, 13\n\t" \ "srl $0, $0, 29\n\t" \ "srl $0, $0, 3\n\t" \ "srl $0, $0, 19\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("move $11, %1\n\t" /*default*/ \ "move $12, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* T3 = client_request ( T4 ) */ \ "or $13, $13, $13\n\t" \ "move %0, $11\n\t" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$11", "$12", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* %t9 = guest_NRADDR */ \ "or $14, $14, $14\n\t" \ "move %0, $11" /*result*/ \ : "=r" (__addr) \ : \ : "$11" \ ); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir *%t9 */ \ "or $15, $15, $15\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or $11, $11, $11\n\t" \ ); \ } while (0) #endif /* PLAT_mips32_linux */ /* ------------------------- mips64-linux ---------------- */ #if defined(PLAT_mips64_linux) typedef struct { unsigned long nraddr; /* where's the code? */ } OrigFn; /* dsll $0,$0, 3 * dsll $0,$0, 13 * dsll $0,$0, 29 * dsll $0,$0, 19*/ #define __SPECIAL_INSTRUCTION_PREAMBLE \ "dsll $0,$0, 3 ; dsll $0,$0,13\n\t" \ "dsll $0,$0,29 ; dsll $0,$0,19\n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned long int _zzq_args[6]; \ volatile unsigned long int _zzq_result; \ _zzq_args[0] = (unsigned long int)(_zzq_request); \ _zzq_args[1] = (unsigned long int)(_zzq_arg1); \ _zzq_args[2] = (unsigned long int)(_zzq_arg2); \ _zzq_args[3] = (unsigned long int)(_zzq_arg3); \ _zzq_args[4] = (unsigned long int)(_zzq_arg4); \ _zzq_args[5] = (unsigned long int)(_zzq_arg5); \ __asm__ volatile("move $11, %1\n\t" /*default*/ \ "move $12, %2\n\t" /*ptr*/ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* $11 = client_request ( $12 ) */ \ "or $13, $13, $13\n\t" \ "move %0, $11\n\t" /*result*/ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$11", "$12", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* $11 = guest_NRADDR */ \ "or $14, $14, $14\n\t" \ "move %0, $11" /*result*/ \ : "=r" (__addr) \ : \ : "$11"); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir $25 */ \ "or $15, $15, $15\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or $11, $11, $11\n\t" \ ); \ } while (0) #endif /* PLAT_mips64_linux */ #if defined(PLAT_nanomips_linux) typedef struct { unsigned int nraddr; /* where's the code? */ } OrigFn; /* 8000 c04d srl zero, zero, 13 8000 c05d srl zero, zero, 29 8000 c043 srl zero, zero, 3 8000 c053 srl zero, zero, 19 */ #define __SPECIAL_INSTRUCTION_PREAMBLE "srl[32] $zero, $zero, 13 \n\t" \ "srl[32] $zero, $zero, 29 \n\t" \ "srl[32] $zero, $zero, 3 \n\t" \ "srl[32] $zero, $zero, 19 \n\t" #define VALGRIND_DO_CLIENT_REQUEST_EXPR( \ _zzq_default, _zzq_request, \ _zzq_arg1, _zzq_arg2, _zzq_arg3, _zzq_arg4, _zzq_arg5) \ __extension__ \ ({ volatile unsigned int _zzq_args[6]; \ volatile unsigned int _zzq_result; \ _zzq_args[0] = (unsigned int)(_zzq_request); \ _zzq_args[1] = (unsigned int)(_zzq_arg1); \ _zzq_args[2] = (unsigned int)(_zzq_arg2); \ _zzq_args[3] = (unsigned int)(_zzq_arg3); \ _zzq_args[4] = (unsigned int)(_zzq_arg4); \ _zzq_args[5] = (unsigned int)(_zzq_arg5); \ __asm__ volatile("move $a7, %1\n\t" /* default */ \ "move $t0, %2\n\t" /* ptr */ \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* $a7 = client_request( $t0 ) */ \ "or[32] $t0, $t0, $t0\n\t" \ "move %0, $a7\n\t" /* result */ \ : "=r" (_zzq_result) \ : "r" (_zzq_default), "r" (&_zzq_args[0]) \ : "$a7", "$t0", "memory"); \ _zzq_result; \ }) #define VALGRIND_GET_NR_CONTEXT(_zzq_rlval) \ { volatile OrigFn* _zzq_orig = &(_zzq_rlval); \ volatile unsigned long int __addr; \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ /* $a7 = guest_NRADDR */ \ "or[32] $t1, $t1, $t1\n\t" \ "move %0, $a7" /*result*/ \ : "=r" (__addr) \ : \ : "$a7"); \ _zzq_orig->nraddr = __addr; \ } #define VALGRIND_CALL_NOREDIR_T9 \ __SPECIAL_INSTRUCTION_PREAMBLE \ /* call-noredir $25 */ \ "or[32] $t2, $t2, $t2\n\t" #define VALGRIND_VEX_INJECT_IR() \ do { \ __asm__ volatile(__SPECIAL_INSTRUCTION_PREAMBLE \ "or[32] $t3, $t3, $t3\n\t" \ ); \ } while (0) #endif /* Insert assembly code for other platforms here... */ #endif /* NVALGRIND */ /* ------------------------------------------------------------------ */ /* PLATFORM SPECIFICS for FUNCTION WRAPPING. This is all very */ /* ugly. It's the least-worst tradeoff I can think of. */ /* ------------------------------------------------------------------ */ /* This section defines magic (a.k.a appalling-hack) macros for doing guaranteed-no-redirection macros, so as to get from function wrappers to the functions they are wrapping. The whole point is to construct standard call sequences, but to do the call itself with a special no-redirect call pseudo-instruction that the JIT understands and handles specially. This section is long and repetitious, and I can't see a way to make it shorter. The naming scheme is as follows: CALL_FN_{W,v}_{v,W,WW,WWW,WWWW,5W,6W,7W,etc} 'W' stands for "word" and 'v' for "void". Hence there are different macros for calling arity 0, 1, 2, 3, 4, etc, functions, and for each, the possibility of returning a word-typed result, or no result. */ /* Use these to write the name of your wrapper. NOTE: duplicates VG_WRAP_FUNCTION_Z{U,Z} in pub_tool_redir.h. NOTE also: inserts the default behaviour equivalance class tag "0000" into the name. See pub_tool_redir.h for details -- normally you don't need to think about this, though. */ /* Use an extra level of macroisation so as to ensure the soname/fnname args are fully macro-expanded before pasting them together. */ #define VG_CONCAT4(_aa,_bb,_cc,_dd) _aa##_bb##_cc##_dd #define I_WRAP_SONAME_FNNAME_ZU(soname,fnname) \ VG_CONCAT4(_vgw00000ZU_,soname,_,fnname) #define I_WRAP_SONAME_FNNAME_ZZ(soname,fnname) \ VG_CONCAT4(_vgw00000ZZ_,soname,_,fnname) /* Use this macro from within a wrapper function to collect the context (address and possibly other info) of the original function. Once you have that you can then use it in one of the CALL_FN_ macros. The type of the argument _lval is OrigFn. */ #define VALGRIND_GET_ORIG_FN(_lval) VALGRIND_GET_NR_CONTEXT(_lval) /* Also provide end-user facilities for function replacement, rather than wrapping. A replacement function differs from a wrapper in that it has no way to get hold of the original function being called, and hence no way to call onwards to it. In a replacement function, VALGRIND_GET_ORIG_FN always returns zero. */ #define I_REPLACE_SONAME_FNNAME_ZU(soname,fnname) \ VG_CONCAT4(_vgr00000ZU_,soname,_,fnname) #define I_REPLACE_SONAME_FNNAME_ZZ(soname,fnname) \ VG_CONCAT4(_vgr00000ZZ_,soname,_,fnname) /* Derivatives of the main macros below, for calling functions returning void. */ #define CALL_FN_v_v(fnptr) \ do { volatile unsigned long _junk; \ CALL_FN_W_v(_junk,fnptr); } while (0) #define CALL_FN_v_W(fnptr, arg1) \ do { volatile unsigned long _junk; \ CALL_FN_W_W(_junk,fnptr,arg1); } while (0) #define CALL_FN_v_WW(fnptr, arg1,arg2) \ do { volatile unsigned long _junk; \ CALL_FN_W_WW(_junk,fnptr,arg1,arg2); } while (0) #define CALL_FN_v_WWW(fnptr, arg1,arg2,arg3) \ do { volatile unsigned long _junk; \ CALL_FN_W_WWW(_junk,fnptr,arg1,arg2,arg3); } while (0) #define CALL_FN_v_WWWW(fnptr, arg1,arg2,arg3,arg4) \ do { volatile unsigned long _junk; \ CALL_FN_W_WWWW(_junk,fnptr,arg1,arg2,arg3,arg4); } while (0) #define CALL_FN_v_5W(fnptr, arg1,arg2,arg3,arg4,arg5) \ do { volatile unsigned long _junk; \ CALL_FN_W_5W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5); } while (0) #define CALL_FN_v_6W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6) \ do { volatile unsigned long _junk; \ CALL_FN_W_6W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6); } while (0) #define CALL_FN_v_7W(fnptr, arg1,arg2,arg3,arg4,arg5,arg6,arg7) \ do { volatile unsigned long _junk; \ CALL_FN_W_7W(_junk,fnptr,arg1,arg2,arg3,arg4,arg5,arg6,arg7); } while (0) /* ----------------- x86-{linux,darwin,solaris} ---------------- */ #if defined(PLAT_x86_linux) || defined(PLAT_x86_darwin) \ || defined(PLAT_x86_solaris) || defined(PLAT_x86_freebsd) /* These regs are trashed by the hidden call. No need to mention eax as gcc can already see that, plus causes gcc to bomb. */ #define __CALLER_SAVED_REGS /*"eax"*/ "ecx", "edx" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "movl %%esp,%%edi\n\t" \ "andl $0xfffffff0,%%esp\n\t" #define VALGRIND_RESTORE_STACK \ "movl %%edi,%%esp\n\t" /* These CALL_FN_ macros assume that on x86-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $12, %%esp\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $8, %%esp\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "subl $4, %%esp\n\t" \ "pushl 44(%%eax)\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "pushl 48(%%eax)\n\t" \ "pushl 44(%%eax)\n\t" \ "pushl 40(%%eax)\n\t" \ "pushl 36(%%eax)\n\t" \ "pushl 32(%%eax)\n\t" \ "pushl 28(%%eax)\n\t" \ "pushl 24(%%eax)\n\t" \ "pushl 20(%%eax)\n\t" \ "pushl 16(%%eax)\n\t" \ "pushl 12(%%eax)\n\t" \ "pushl 8(%%eax)\n\t" \ "pushl 4(%%eax)\n\t" \ "movl (%%eax), %%eax\n\t" /* target->%eax */ \ VALGRIND_CALL_NOREDIR_EAX \ VALGRIND_RESTORE_STACK \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "edi" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_x86_linux || PLAT_x86_darwin || PLAT_x86_solaris */ /* ---------------- amd64-{linux,darwin,solaris} --------------- */ #if defined(PLAT_amd64_linux) || defined(PLAT_amd64_darwin) \ || defined(PLAT_amd64_solaris) || defined(PLAT_amd64_freebsd) /* ARGREGS: rdi rsi rdx rcx r8 r9 (the rest on stack in R-to-L order) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS /*"rax",*/ "rcx", "rdx", "rsi", \ "rdi", "r8", "r9", "r10", "r11" /* This is all pretty complex. It's so as to make stack unwinding work reliably. See bug 243270. The basic problem is the sub and add of 128 of %rsp in all of the following macros. If gcc believes the CFA is in %rsp, then unwinding may fail, because what's at the CFA is not what gcc "expected" when it constructs the CFIs for the places where the macros are instantiated. But we can't just add a CFI annotation to increase the CFA offset by 128, to match the sub of 128 from %rsp, because we don't know whether gcc has chosen %rsp as the CFA at that point, or whether it has chosen some other register (eg, %rbp). In the latter case, adding a CFI annotation to change the CFA offset is simply wrong. So the solution is to get hold of the CFA using __builtin_dwarf_cfa(), put it in a known register, and add a CFI annotation to say what the register is. We choose %rbp for this (perhaps perversely), because: (1) %rbp is already subject to unwinding. If a new register was chosen then the unwinder would have to unwind it in all stack traces, which is expensive, and (2) %rbp is already subject to precise exception updates in the JIT. If a new register was chosen, we'd have to have precise exceptions for it too, which reduces performance of the generated code. However .. one extra complication. We can't just whack the result of __builtin_dwarf_cfa() into %rbp and then add %rbp to the list of trashed registers at the end of the inline assembly fragments; gcc won't allow %rbp to appear in that list. Hence instead we need to stash %rbp in %r15 for the duration of the asm, and say that %r15 is trashed instead. gcc seems happy to go with that. Oh .. and this all needs to be conditionalised so that it is unchanged from before this commit, when compiled with older gccs that don't support __builtin_dwarf_cfa. Furthermore, since this header file is freestanding, it has to be independent of config.h, and so the following conditionalisation cannot depend on configure time checks. Although it's not clear from 'defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM)', this expression excludes Darwin. .cfi directives in Darwin assembly appear to be completely different and I haven't investigated how they work. For even more entertainment value, note we have to use the completely undocumented __builtin_dwarf_cfa(), which appears to really compute the CFA, whereas __builtin_frame_address(0) claims to but actually doesn't. See https://bugs.kde.org/show_bug.cgi?id=243270#c47 */ #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM) # define __FRAME_POINTER \ ,"r"(__builtin_dwarf_cfa()) # define VALGRIND_CFI_PROLOGUE \ "movq %%rbp, %%r15\n\t" \ "movq %2, %%rbp\n\t" \ ".cfi_remember_state\n\t" \ ".cfi_def_cfa rbp, 0\n\t" # define VALGRIND_CFI_EPILOGUE \ "movq %%r15, %%rbp\n\t" \ ".cfi_restore_state\n\t" #else # define __FRAME_POINTER # define VALGRIND_CFI_PROLOGUE # define VALGRIND_CFI_EPILOGUE #endif /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "movq %%rsp,%%r14\n\t" \ "andq $0xfffffffffffffff0,%%rsp\n\t" #define VALGRIND_RESTORE_STACK \ "movq %%r14,%%rsp\n\t" /* These CALL_FN_ macros assume that on amd64-linux, sizeof(unsigned long) == 8. */ /* NB 9 Sept 07. There is a nasty kludge here in all these CALL_FN_ macros. In order not to trash the stack redzone, we need to drop %rsp by 128 before the hidden call, and restore afterwards. The nastyness is that it is only by luck that the stack still appears to be unwindable during the hidden call - since then the behaviour of any routine using this macro does not match what the CFI data says. Sigh. Why is this important? Imagine that a wrapper has a stack allocated local, and passes to the hidden call, a pointer to it. Because gcc does not know about the hidden call, it may allocate that local in the redzone. Unfortunately the hidden call may then trash it before it comes to use it. So we must step clear of the redzone, for the duration of the hidden call, to make it safe. Probably the same problem afflicts the other redzone-style ABIs too (ppc64-linux); but for those, the stack is self describing (none of this CFI nonsense) so at least messing with the stack pointer doesn't give a danger of non-unwindable stack. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $136,%%rsp\n\t" \ "pushq 88(%%rax)\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ VALGRIND_ALIGN_STACK \ "subq $128,%%rsp\n\t" \ "pushq 96(%%rax)\n\t" \ "pushq 88(%%rax)\n\t" \ "pushq 80(%%rax)\n\t" \ "pushq 72(%%rax)\n\t" \ "pushq 64(%%rax)\n\t" \ "pushq 56(%%rax)\n\t" \ "movq 48(%%rax), %%r9\n\t" \ "movq 40(%%rax), %%r8\n\t" \ "movq 32(%%rax), %%rcx\n\t" \ "movq 24(%%rax), %%rdx\n\t" \ "movq 16(%%rax), %%rsi\n\t" \ "movq 8(%%rax), %%rdi\n\t" \ "movq (%%rax), %%rax\n\t" /* target->%rax */ \ VALGRIND_CALL_NOREDIR_RAX \ VALGRIND_RESTORE_STACK \ VALGRIND_CFI_EPILOGUE \ : /*out*/ "=a" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r14", "r15" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_amd64_linux || PLAT_amd64_darwin || PLAT_amd64_solaris */ /* ------------------------ ppc32-linux ------------------------ */ #if defined(PLAT_ppc32_linux) /* This is useful for finding out about the on-stack stuff: extern int f9 ( int,int,int,int,int,int,int,int,int ); extern int f10 ( int,int,int,int,int,int,int,int,int,int ); extern int f11 ( int,int,int,int,int,int,int,int,int,int,int ); extern int f12 ( int,int,int,int,int,int,int,int,int,int,int,int ); int g9 ( void ) { return f9(11,22,33,44,55,66,77,88,99); } int g10 ( void ) { return f10(11,22,33,44,55,66,77,88,99,110); } int g11 ( void ) { return f11(11,22,33,44,55,66,77,88,99,110,121); } int g12 ( void ) { return f12(11,22,33,44,55,66,77,88,99,110,121,132); } */ /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r2", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rlwinm 1,1,0,0,27\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc32-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-16\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-16\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-32\n\t" \ /* arg11 */ \ "lwz 3,44(11)\n\t" \ "stw 3,16(1)\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ _argvec[12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "addi 1,1,-32\n\t" \ /* arg12 */ \ "lwz 3,48(11)\n\t" \ "stw 3,20(1)\n\t" \ /* arg11 */ \ "lwz 3,44(11)\n\t" \ "stw 3,16(1)\n\t" \ /* arg10 */ \ "lwz 3,40(11)\n\t" \ "stw 3,12(1)\n\t" \ /* arg9 */ \ "lwz 3,36(11)\n\t" \ "stw 3,8(1)\n\t" \ /* args1-8 */ \ "lwz 3,4(11)\n\t" /* arg1->r3 */ \ "lwz 4,8(11)\n\t" \ "lwz 5,12(11)\n\t" \ "lwz 6,16(11)\n\t" /* arg4->r6 */ \ "lwz 7,20(11)\n\t" \ "lwz 8,24(11)\n\t" \ "lwz 9,28(11)\n\t" \ "lwz 10,32(11)\n\t" /* arg8->r10 */ \ "lwz 11,0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ VALGRIND_RESTORE_STACK \ "mr %0,3" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc32_linux */ /* ------------------------ ppc64-linux ------------------------ */ #if defined(PLAT_ppc64be_linux) /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rldicr 1,1,0,59\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+0]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+1]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+2]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+3]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+4]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+5]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+6]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+7]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+8]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+9]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+10]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+11]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg11 */ \ "ld 3,88(11)\n\t" \ "std 3,128(1)\n\t" \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+12]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ _argvec[2+12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 11,%1\n\t" \ "std 2,-16(11)\n\t" /* save tocptr */ \ "ld 2,-8(11)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg12 */ \ "ld 3,96(11)\n\t" \ "std 3,136(1)\n\t" \ /* arg11 */ \ "ld 3,88(11)\n\t" \ "std 3,128(1)\n\t" \ /* arg10 */ \ "ld 3,80(11)\n\t" \ "std 3,120(1)\n\t" \ /* arg9 */ \ "ld 3,72(11)\n\t" \ "std 3,112(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(11)\n\t" /* arg1->r3 */ \ "ld 4, 16(11)\n\t" /* arg2->r4 */ \ "ld 5, 24(11)\n\t" /* arg3->r5 */ \ "ld 6, 32(11)\n\t" /* arg4->r6 */ \ "ld 7, 40(11)\n\t" /* arg5->r7 */ \ "ld 8, 48(11)\n\t" /* arg6->r8 */ \ "ld 9, 56(11)\n\t" /* arg7->r9 */ \ "ld 10, 64(11)\n\t" /* arg8->r10 */ \ "ld 11, 0(11)\n\t" /* target->r11 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R11 \ "mr 11,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(11)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc64be_linux */ /* ------------------------- ppc64le-linux ----------------------- */ #if defined(PLAT_ppc64le_linux) /* ARGREGS: r3 r4 r5 r6 r7 r8 r9 r10 (the rest on stack somewhere) */ /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "lr", "ctr", "xer", \ "cr0", "cr1", "cr2", "cr3", "cr4", "cr5", "cr6", "cr7", \ "r0", "r3", "r4", "r5", "r6", "r7", "r8", "r9", "r10", \ "r11", "r12", "r13" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ #define VALGRIND_ALIGN_STACK \ "mr 28,1\n\t" \ "rldicr 1,1,0,59\n\t" #define VALGRIND_RESTORE_STACK \ "mr 1,28\n\t" /* These CALL_FN_ macros assume that on ppc64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+0]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+1]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+2]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+3]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+4]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+5]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+6]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+7]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+8]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+9]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+10]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-128\n\t" /* expand stack frame */ \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+11]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg11 */ \ "ld 3,88(12)\n\t" \ "std 3,112(1)\n\t" \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3+12]; \ volatile unsigned long _res; \ /* _argvec[0] holds current r2 across the call */ \ _argvec[1] = (unsigned long)_orig.r2; \ _argvec[2] = (unsigned long)_orig.nraddr; \ _argvec[2+1] = (unsigned long)arg1; \ _argvec[2+2] = (unsigned long)arg2; \ _argvec[2+3] = (unsigned long)arg3; \ _argvec[2+4] = (unsigned long)arg4; \ _argvec[2+5] = (unsigned long)arg5; \ _argvec[2+6] = (unsigned long)arg6; \ _argvec[2+7] = (unsigned long)arg7; \ _argvec[2+8] = (unsigned long)arg8; \ _argvec[2+9] = (unsigned long)arg9; \ _argvec[2+10] = (unsigned long)arg10; \ _argvec[2+11] = (unsigned long)arg11; \ _argvec[2+12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "mr 12,%1\n\t" \ "std 2,-16(12)\n\t" /* save tocptr */ \ "ld 2,-8(12)\n\t" /* use nraddr's tocptr */ \ "addi 1,1,-144\n\t" /* expand stack frame */ \ /* arg12 */ \ "ld 3,96(12)\n\t" \ "std 3,120(1)\n\t" \ /* arg11 */ \ "ld 3,88(12)\n\t" \ "std 3,112(1)\n\t" \ /* arg10 */ \ "ld 3,80(12)\n\t" \ "std 3,104(1)\n\t" \ /* arg9 */ \ "ld 3,72(12)\n\t" \ "std 3,96(1)\n\t" \ /* args1-8 */ \ "ld 3, 8(12)\n\t" /* arg1->r3 */ \ "ld 4, 16(12)\n\t" /* arg2->r4 */ \ "ld 5, 24(12)\n\t" /* arg3->r5 */ \ "ld 6, 32(12)\n\t" /* arg4->r6 */ \ "ld 7, 40(12)\n\t" /* arg5->r7 */ \ "ld 8, 48(12)\n\t" /* arg6->r8 */ \ "ld 9, 56(12)\n\t" /* arg7->r9 */ \ "ld 10, 64(12)\n\t" /* arg8->r10 */ \ "ld 12, 0(12)\n\t" /* target->r12 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R12 \ "mr 12,%1\n\t" \ "mr %0,3\n\t" \ "ld 2,-16(12)\n\t" /* restore tocptr */ \ VALGRIND_RESTORE_STACK \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[2]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r28" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_ppc64le_linux */ /* ------------------------- arm-linux ------------------------- */ #if defined(PLAT_arm_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "r0", "r1", "r2", "r3","r4", "r12", "r14" /* Macros to save and align the stack before making a function call and restore it afterwards as gcc may not keep the stack pointer aligned if it doesn't realise calls are being made to other functions. */ /* This is a bit tricky. We store the original stack pointer in r10 as it is callee-saves. gcc doesn't allow the use of r11 for some reason. Also, we can't directly "bic" the stack pointer in thumb mode since r13 isn't an allowed register number in that context. So use r4 as a temporary, since that is about to get trashed anyway, just after each use of this macro. Side effect is we need to be very careful about any future changes, since VALGRIND_ALIGN_STACK simply assumes r4 is usable. */ #define VALGRIND_ALIGN_STACK \ "mov r10, sp\n\t" \ "mov r4, sp\n\t" \ "bic r4, r4, #7\n\t" \ "mov sp, r4\n\t" #define VALGRIND_RESTORE_STACK \ "mov sp, r10\n\t" /* These CALL_FN_ macros assume that on arm-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "push {r0} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "push {r0, r1} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "push {r0, r1, r2} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "push {r0, r1, r2, r3} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #40] \n\t" \ "push {r0} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #4 \n\t" \ "ldr r0, [%1, #40] \n\t" \ "ldr r1, [%1, #44] \n\t" \ "push {r0, r1} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr r0, [%1, #40] \n\t" \ "ldr r1, [%1, #44] \n\t" \ "ldr r2, [%1, #48] \n\t" \ "push {r0, r1, r2} \n\t" \ "ldr r0, [%1, #20] \n\t" \ "ldr r1, [%1, #24] \n\t" \ "ldr r2, [%1, #28] \n\t" \ "ldr r3, [%1, #32] \n\t" \ "ldr r4, [%1, #36] \n\t" \ "push {r0, r1, r2, r3, r4} \n\t" \ "ldr r0, [%1, #4] \n\t" \ "ldr r1, [%1, #8] \n\t" \ "ldr r2, [%1, #12] \n\t" \ "ldr r3, [%1, #16] \n\t" \ "ldr r4, [%1] \n\t" /* target->r4 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_R4 \ VALGRIND_RESTORE_STACK \ "mov %0, r0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "r10" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_arm_linux */ /* ------------------------ arm64-linux ------------------------ */ #if defined(PLAT_arm64_linux) || defined(PLAT_arm64_freebsd) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS \ "x0", "x1", "x2", "x3","x4", "x5", "x6", "x7", "x8", "x9", \ "x10", "x11", "x12", "x13", "x14", "x15", "x16", "x17", \ "x18", "x19", "x20", "x30", \ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", "v8", "v9", \ "v10", "v11", "v12", "v13", "v14", "v15", "v16", "v17", \ "v18", "v19", "v20", "v21", "v22", "v23", "v24", "v25", \ "v26", "v27", "v28", "v29", "v30", "v31" /* x21 is callee-saved, so we can use it to save and restore SP around the hidden call. */ #define VALGRIND_ALIGN_STACK \ "mov x21, sp\n\t" \ "bic sp, x21, #15\n\t" #define VALGRIND_RESTORE_STACK \ "mov sp, x21\n\t" /* These CALL_FN_ macros assume that on arm64-linux, sizeof(unsigned long) == 8. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x20 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x20 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x30 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1, #88] \n\t" \ "str x8, [sp, #16] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10,arg11, \ arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ VALGRIND_ALIGN_STACK \ "sub sp, sp, #0x30 \n\t" \ "ldr x0, [%1, #8] \n\t" \ "ldr x1, [%1, #16] \n\t" \ "ldr x2, [%1, #24] \n\t" \ "ldr x3, [%1, #32] \n\t" \ "ldr x4, [%1, #40] \n\t" \ "ldr x5, [%1, #48] \n\t" \ "ldr x6, [%1, #56] \n\t" \ "ldr x7, [%1, #64] \n\t" \ "ldr x8, [%1, #72] \n\t" \ "str x8, [sp, #0] \n\t" \ "ldr x8, [%1, #80] \n\t" \ "str x8, [sp, #8] \n\t" \ "ldr x8, [%1, #88] \n\t" \ "str x8, [sp, #16] \n\t" \ "ldr x8, [%1, #96] \n\t" \ "str x8, [sp, #24] \n\t" \ "ldr x8, [%1] \n\t" /* target->x8 */ \ VALGRIND_BRANCH_AND_LINK_TO_NOREDIR_X8 \ VALGRIND_RESTORE_STACK \ "mov %0, x0" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS, "x21" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_arm64_linux */ /* ------------------------- s390x-linux ------------------------- */ #if defined(PLAT_s390x_linux) /* Similar workaround as amd64 (see above), but we use r11 as frame pointer and save the old r11 in r7. r11 might be used for argvec, therefore we copy argvec in r1 since r1 is clobbered after the call anyway. */ #if defined(__GNUC__) && defined(__GCC_HAVE_DWARF2_CFI_ASM) # define __FRAME_POINTER \ ,"d"(__builtin_dwarf_cfa()) # define VALGRIND_CFI_PROLOGUE \ ".cfi_remember_state\n\t" \ "lgr 1,%1\n\t" /* copy the argvec pointer in r1 */ \ "lgr 7,11\n\t" \ "lgr 11,%2\n\t" \ ".cfi_def_cfa 11, 0\n\t" # define VALGRIND_CFI_EPILOGUE \ "lgr 11, 7\n\t" \ ".cfi_restore_state\n\t" #else # define __FRAME_POINTER # define VALGRIND_CFI_PROLOGUE \ "lgr 1,%1\n\t" # define VALGRIND_CFI_EPILOGUE #endif /* Nb: On s390 the stack pointer is properly aligned *at all times* according to the s390 GCC maintainer. (The ABI specification is not precise in this regard.) Therefore, VALGRIND_ALIGN_STACK and VALGRIND_RESTORE_STACK are not defined here. */ /* These regs are trashed by the hidden call. Note that we overwrite r14 in s390_irgen_noredir (VEX/priv/guest_s390_irgen.c) to give the function a proper return address. All others are ABI defined call clobbers. */ #if defined(__VX__) || defined(__S390_VX__) #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14", \ "v0", "v1", "v2", "v3", "v4", "v5", "v6", "v7", \ "v8", "v9", "v10", "v11", "v12", "v13", "v14", "v15", \ "v16", "v17", "v18", "v19", "v20", "v21", "v22", "v23", \ "v24", "v25", "v26", "v27", "v28", "v29", "v30", "v31" #else #define __CALLER_SAVED_REGS "0", "1", "2", "3", "4", "5", "14", \ "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" #endif /* Nb: Although r11 is modified in the asm snippets below (inside VALGRIND_CFI_PROLOGUE) it is not listed in the clobber section, for two reasons: (1) r11 is restored in VALGRIND_CFI_EPILOGUE, so effectively it is not modified (2) GCC will complain that r11 cannot appear inside a clobber section, when compiled with -O -fno-omit-frame-pointer */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 1, 0(1)\n\t" /* target->r1 */ \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "d" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) /* The call abi has the arguments in r2-r6 and stack */ #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1, arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1, arg2, arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1, arg2, arg3, arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1, arg2, arg3, arg4, arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-160\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,160\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-168\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,168\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-176\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,176\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-184\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,184\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-192\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,192\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-200\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,200\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10, arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-208\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "mvc 200(8,15), 88(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,208\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1, arg2, arg3, arg4, arg5, \ arg6, arg7 ,arg8, arg9, arg10, arg11, arg12)\ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)arg1; \ _argvec[2] = (unsigned long)arg2; \ _argvec[3] = (unsigned long)arg3; \ _argvec[4] = (unsigned long)arg4; \ _argvec[5] = (unsigned long)arg5; \ _argvec[6] = (unsigned long)arg6; \ _argvec[7] = (unsigned long)arg7; \ _argvec[8] = (unsigned long)arg8; \ _argvec[9] = (unsigned long)arg9; \ _argvec[10] = (unsigned long)arg10; \ _argvec[11] = (unsigned long)arg11; \ _argvec[12] = (unsigned long)arg12; \ __asm__ volatile( \ VALGRIND_CFI_PROLOGUE \ "aghi 15,-216\n\t" \ "lg 2, 8(1)\n\t" \ "lg 3,16(1)\n\t" \ "lg 4,24(1)\n\t" \ "lg 5,32(1)\n\t" \ "lg 6,40(1)\n\t" \ "mvc 160(8,15), 48(1)\n\t" \ "mvc 168(8,15), 56(1)\n\t" \ "mvc 176(8,15), 64(1)\n\t" \ "mvc 184(8,15), 72(1)\n\t" \ "mvc 192(8,15), 80(1)\n\t" \ "mvc 200(8,15), 88(1)\n\t" \ "mvc 208(8,15), 96(1)\n\t" \ "lg 1, 0(1)\n\t" \ VALGRIND_CALL_NOREDIR_R1 \ "aghi 15,216\n\t" \ VALGRIND_CFI_EPILOGUE \ "lgr %0, 2\n\t" \ : /*out*/ "=d" (_res) \ : /*in*/ "a" (&_argvec[0]) __FRAME_POINTER \ : /*trash*/ "cc", "memory", __CALLER_SAVED_REGS,"6","7" \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_s390x_linux */ /* ------------------------- mips32-linux ----------------------- */ #if defined(PLAT_mips32_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \ "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \ "$25", "$31" /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16\n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" /* arg1*/ \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "subu $29, $29, 16 \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 16 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 24\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 24 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 32\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "nop\n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 32 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 32\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 32 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 40\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 40 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 40\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 40 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 48\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 48 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 48\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 44(%1) \n\t" \ "sw $4, 40($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 48 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ "subu $29, $29, 8 \n\t" \ "sw $28, 0($29) \n\t" \ "sw $31, 4($29) \n\t" \ "lw $4, 20(%1) \n\t" \ "subu $29, $29, 56\n\t" \ "sw $4, 16($29) \n\t" \ "lw $4, 24(%1) \n\t" \ "sw $4, 20($29) \n\t" \ "lw $4, 28(%1) \n\t" \ "sw $4, 24($29) \n\t" \ "lw $4, 32(%1) \n\t" \ "sw $4, 28($29) \n\t" \ "lw $4, 36(%1) \n\t" \ "sw $4, 32($29) \n\t" \ "lw $4, 40(%1) \n\t" \ "sw $4, 36($29) \n\t" \ "lw $4, 44(%1) \n\t" \ "sw $4, 40($29) \n\t" \ "lw $4, 48(%1) \n\t" \ "sw $4, 44($29) \n\t" \ "lw $4, 4(%1) \n\t" \ "lw $5, 8(%1) \n\t" \ "lw $6, 12(%1) \n\t" \ "lw $7, 16(%1) \n\t" \ "lw $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "addu $29, $29, 56 \n\t" \ "lw $28, 0($29) \n\t" \ "lw $31, 4($29) \n\t" \ "addu $29, $29, 8 \n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_mips32_linux */ /* ------------------------- nanomips-linux -------------------- */ #if defined(PLAT_nanomips_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$t4", "$t5", "$a0", "$a1", "$a2", \ "$a3", "$a4", "$a5", "$a6", "$a7", "$t0", "$t1", "$t2", "$t3", \ "$t8","$t9", "$at" /* These CALL_FN_ macros assume that on mips-linux, sizeof(unsigned long) == 4. */ #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[1]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[2]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[3]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[4]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[5]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[6]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[7]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[8]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ "lw $a6,28(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[9]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ __asm__ volatile( \ "lw $t9, 0(%1)\n\t" \ "lw $a0, 4(%1)\n\t" \ "lw $a1, 8(%1)\n\t" \ "lw $a2,12(%1)\n\t" \ "lw $a3,16(%1)\n\t" \ "lw $a4,20(%1)\n\t" \ "lw $a5,24(%1)\n\t" \ "lw $a6,28(%1)\n\t" \ "lw $a7,32(%1)\n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[10]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[11]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[12]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9,44(%1) \n\t" \ "sw $t9, 8($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long _argvec[13]; \ volatile unsigned long _res; \ _argvec[0] = (unsigned long)_orig.nraddr; \ _argvec[1] = (unsigned long)(arg1); \ _argvec[2] = (unsigned long)(arg2); \ _argvec[3] = (unsigned long)(arg3); \ _argvec[4] = (unsigned long)(arg4); \ _argvec[5] = (unsigned long)(arg5); \ _argvec[6] = (unsigned long)(arg6); \ _argvec[7] = (unsigned long)(arg7); \ _argvec[8] = (unsigned long)(arg8); \ _argvec[9] = (unsigned long)(arg9); \ _argvec[10] = (unsigned long)(arg10); \ _argvec[11] = (unsigned long)(arg11); \ _argvec[12] = (unsigned long)(arg12); \ __asm__ volatile( \ "addiu $sp, $sp, -16 \n\t" \ "lw $t9,36(%1) \n\t" \ "sw $t9, 0($sp) \n\t" \ "lw $t9,40(%1) \n\t" \ "sw $t9, 4($sp) \n\t" \ "lw $t9,44(%1) \n\t" \ "sw $t9, 8($sp) \n\t" \ "lw $t9,48(%1) \n\t" \ "sw $t9,12($sp) \n\t" \ "lw $t9, 0(%1) \n\t" \ "lw $a0, 4(%1) \n\t" \ "lw $a1, 8(%1) \n\t" \ "lw $a2,12(%1) \n\t" \ "lw $a3,16(%1) \n\t" \ "lw $a4,20(%1) \n\t" \ "lw $a5,24(%1) \n\t" \ "lw $a6,28(%1) \n\t" \ "lw $a7,32(%1) \n\t" \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $a0 \n\t" \ "addiu $sp, $sp, 16 \n\t" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) _res; \ } while (0) #endif /* PLAT_nanomips_linux */ /* ------------------------- mips64-linux ------------------------- */ #if defined(PLAT_mips64_linux) /* These regs are trashed by the hidden call. */ #define __CALLER_SAVED_REGS "$2", "$3", "$4", "$5", "$6", \ "$7", "$8", "$9", "$10", "$11", "$12", "$13", "$14", "$15", "$24", \ "$25", "$31" /* These CALL_FN_ macros assume that on mips64-linux, sizeof(long long) == 8. */ #define MIPS64_LONG2REG_CAST(x) ((long long)(long)x) #define CALL_FN_W_v(lval, orig) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[1]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ __asm__ volatile( \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "0" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_W(lval, orig, arg1) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[2]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" /* arg1*/ \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WW(lval, orig, arg1,arg2) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[3]; \ volatile unsigned long long _res; \ _argvec[0] = _orig.nraddr; \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WWW(lval, orig, arg1,arg2,arg3) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[4]; \ volatile unsigned long long _res; \ _argvec[0] = _orig.nraddr; \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_WWWW(lval, orig, arg1,arg2,arg3,arg4) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[5]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_5W(lval, orig, arg1,arg2,arg3,arg4,arg5) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[6]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_6W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[7]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_7W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[8]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_8W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[9]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ __asm__ volatile( \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1) \n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_9W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[10]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ __asm__ volatile( \ "dsubu $29, $29, 8\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 8\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_10W(lval, orig, arg1,arg2,arg3,arg4,arg5,arg6, \ arg7,arg8,arg9,arg10) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[11]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ __asm__ volatile( \ "dsubu $29, $29, 16\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 16\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_11W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[12]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ _argvec[11] = MIPS64_LONG2REG_CAST(arg11); \ __asm__ volatile( \ "dsubu $29, $29, 24\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 88(%1)\n\t" \ "sd $4, 16($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 24\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #define CALL_FN_W_12W(lval, orig, arg1,arg2,arg3,arg4,arg5, \ arg6,arg7,arg8,arg9,arg10, \ arg11,arg12) \ do { \ volatile OrigFn _orig = (orig); \ volatile unsigned long long _argvec[13]; \ volatile unsigned long long _res; \ _argvec[0] = MIPS64_LONG2REG_CAST(_orig.nraddr); \ _argvec[1] = MIPS64_LONG2REG_CAST(arg1); \ _argvec[2] = MIPS64_LONG2REG_CAST(arg2); \ _argvec[3] = MIPS64_LONG2REG_CAST(arg3); \ _argvec[4] = MIPS64_LONG2REG_CAST(arg4); \ _argvec[5] = MIPS64_LONG2REG_CAST(arg5); \ _argvec[6] = MIPS64_LONG2REG_CAST(arg6); \ _argvec[7] = MIPS64_LONG2REG_CAST(arg7); \ _argvec[8] = MIPS64_LONG2REG_CAST(arg8); \ _argvec[9] = MIPS64_LONG2REG_CAST(arg9); \ _argvec[10] = MIPS64_LONG2REG_CAST(arg10); \ _argvec[11] = MIPS64_LONG2REG_CAST(arg11); \ _argvec[12] = MIPS64_LONG2REG_CAST(arg12); \ __asm__ volatile( \ "dsubu $29, $29, 32\n\t" \ "ld $4, 72(%1)\n\t" \ "sd $4, 0($29)\n\t" \ "ld $4, 80(%1)\n\t" \ "sd $4, 8($29)\n\t" \ "ld $4, 88(%1)\n\t" \ "sd $4, 16($29)\n\t" \ "ld $4, 96(%1)\n\t" \ "sd $4, 24($29)\n\t" \ "ld $4, 8(%1)\n\t" \ "ld $5, 16(%1)\n\t" \ "ld $6, 24(%1)\n\t" \ "ld $7, 32(%1)\n\t" \ "ld $8, 40(%1)\n\t" \ "ld $9, 48(%1)\n\t" \ "ld $10, 56(%1)\n\t" \ "ld $11, 64(%1)\n\t" \ "ld $25, 0(%1)\n\t" /* target->t9 */ \ VALGRIND_CALL_NOREDIR_T9 \ "daddu $29, $29, 32\n\t" \ "move %0, $2\n" \ : /*out*/ "=r" (_res) \ : /*in*/ "r" (&_argvec[0]) \ : /*trash*/ "memory", __CALLER_SAVED_REGS \ ); \ lval = (__typeof__(lval)) (long)_res; \ } while (0) #endif /* PLAT_mips64_linux */ /* ------------------------------------------------------------------ */ /* ARCHITECTURE INDEPENDENT MACROS for CLIENT REQUESTS. */ /* */ /* ------------------------------------------------------------------ */ /* Some request codes. There are many more of these, but most are not exposed to end-user view. These are the public ones, all of the form 0x1000 + small_number. Core ones are in the range 0x00000000--0x0000ffff. The non-public ones start at 0x2000. */ /* These macros are used by tools -- they must be public, but don't embed them into other programs. */ #define VG_USERREQ_TOOL_BASE(a,b) \ ((unsigned int)(((a)&0xff) << 24 | ((b)&0xff) << 16)) #define VG_IS_TOOL_USERREQ(a, b, v) \ (VG_USERREQ_TOOL_BASE(a,b) == ((v) & 0xffff0000)) /* !! ABIWARNING !! ABIWARNING !! ABIWARNING !! ABIWARNING !! This enum comprises an ABI exported by Valgrind to programs which use client requests. DO NOT CHANGE THE NUMERIC VALUES OF THESE ENTRIES, NOR DELETE ANY -- add new ones at the end of the most relevant group. */ typedef enum { VG_USERREQ__RUNNING_ON_VALGRIND = 0x1001, VG_USERREQ__DISCARD_TRANSLATIONS = 0x1002, /* These allow any function to be called from the simulated CPU but run on the real CPU. Nb: the first arg passed to the function is always the ThreadId of the running thread! So CLIENT_CALL0 actually requires a 1 arg function, etc. */ VG_USERREQ__CLIENT_CALL0 = 0x1101, VG_USERREQ__CLIENT_CALL1 = 0x1102, VG_USERREQ__CLIENT_CALL2 = 0x1103, VG_USERREQ__CLIENT_CALL3 = 0x1104, /* Can be useful in regression testing suites -- eg. can send Valgrind's output to /dev/null and still count errors. */ VG_USERREQ__COUNT_ERRORS = 0x1201, /* Allows the client program and/or gdbserver to execute a monitor command. */ VG_USERREQ__GDB_MONITOR_COMMAND = 0x1202, /* Allows the client program to change a dynamic command line option. */ VG_USERREQ__CLO_CHANGE = 0x1203, /* These are useful and can be interpreted by any tool that tracks malloc() et al, by using vg_replace_malloc.c. */ VG_USERREQ__MALLOCLIKE_BLOCK = 0x1301, VG_USERREQ__RESIZEINPLACE_BLOCK = 0x130b, VG_USERREQ__FREELIKE_BLOCK = 0x1302, /* Memory pool support. */ VG_USERREQ__CREATE_MEMPOOL = 0x1303, VG_USERREQ__DESTROY_MEMPOOL = 0x1304, VG_USERREQ__MEMPOOL_ALLOC = 0x1305, VG_USERREQ__MEMPOOL_FREE = 0x1306, VG_USERREQ__MEMPOOL_TRIM = 0x1307, VG_USERREQ__MOVE_MEMPOOL = 0x1308, VG_USERREQ__MEMPOOL_CHANGE = 0x1309, VG_USERREQ__MEMPOOL_EXISTS = 0x130a, /* Allow printfs to valgrind log. */ /* The first two pass the va_list argument by value, which assumes it is the same size as or smaller than a UWord, which generally isn't the case. Hence are deprecated. The second two pass the vargs by reference and so are immune to this problem. */ /* both :: char* fmt, va_list vargs (DEPRECATED) */ VG_USERREQ__PRINTF = 0x1401, VG_USERREQ__PRINTF_BACKTRACE = 0x1402, /* both :: char* fmt, va_list* vargs */ VG_USERREQ__PRINTF_VALIST_BY_REF = 0x1403, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF = 0x1404, /* Stack support. */ VG_USERREQ__STACK_REGISTER = 0x1501, VG_USERREQ__STACK_DEREGISTER = 0x1502, VG_USERREQ__STACK_CHANGE = 0x1503, /* Wine support */ VG_USERREQ__LOAD_PDB_DEBUGINFO = 0x1601, /* Querying of debug info. */ VG_USERREQ__MAP_IP_TO_SRCLOC = 0x1701, /* Disable/enable error reporting level. Takes a single Word arg which is the delta to this thread's error disablement indicator. Hence 1 disables or further disables errors, and -1 moves back towards enablement. Other values are not allowed. */ VG_USERREQ__CHANGE_ERR_DISABLEMENT = 0x1801, /* Some requests used for Valgrind internal, such as self-test or self-hosting. */ /* Initialise IR injection */ VG_USERREQ__VEX_INIT_FOR_IRI = 0x1901, /* Used by Inner Valgrind to inform Outer Valgrind where to find the list of inner guest threads */ VG_USERREQ__INNER_THREADS = 0x1902 } Vg_ClientRequest; #if !defined(__GNUC__) # define __extension__ /* */ #endif /* Returns the number of Valgrinds this code is running under. That is, 0 if running natively, 1 if running under Valgrind, 2 if running under Valgrind which is running under another Valgrind, etc. */ #define RUNNING_ON_VALGRIND \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* if not */, \ VG_USERREQ__RUNNING_ON_VALGRIND, \ 0, 0, 0, 0, 0) \ /* Discard translation of code in the range [_qzz_addr .. _qzz_addr + _qzz_len - 1]. Useful if you are debugging a JITter or some such, since it provides a way to make sure valgrind will retranslate the invalidated area. Returns no value. */ #define VALGRIND_DISCARD_TRANSLATIONS(_qzz_addr,_qzz_len) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DISCARD_TRANSLATIONS, \ _qzz_addr, _qzz_len, 0, 0, 0) #define VALGRIND_INNER_THREADS(_qzz_addr) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__INNER_THREADS, \ _qzz_addr, 0, 0, 0, 0) /* These requests are for getting Valgrind itself to print something. Possibly with a backtrace. This is a really ugly hack. The return value is the number of characters printed, excluding the "**** " part at the start and the backtrace (if present). */ #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER) /* Modern GCC will optimize the static routine out if unused, and unused attribute will shut down warnings about it. */ static int VALGRIND_PRINTF(const char *format, ...) __attribute__((format(__printf__, 1, 2), __unused__)); #endif static int #if defined(_MSC_VER) __inline #endif VALGRIND_PRINTF(const char *format, ...) { #if defined(NVALGRIND) (void)format; return 0; #else /* NVALGRIND */ #if defined(_MSC_VER) || defined(__MINGW64__) uintptr_t _qzz_res; #else unsigned long _qzz_res; #endif va_list vargs; va_start(vargs, format); #if defined(_MSC_VER) || defined(__MINGW64__) _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_VALIST_BY_REF, (uintptr_t)format, (uintptr_t)&vargs, 0, 0, 0); #else _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_VALIST_BY_REF, (unsigned long)format, (unsigned long)&vargs, 0, 0, 0); #endif va_end(vargs); return (int)_qzz_res; #endif /* NVALGRIND */ } #if defined(__GNUC__) || defined(__INTEL_COMPILER) && !defined(_MSC_VER) static int VALGRIND_PRINTF_BACKTRACE(const char *format, ...) __attribute__((format(__printf__, 1, 2), __unused__)); #endif static int #if defined(_MSC_VER) __inline #endif VALGRIND_PRINTF_BACKTRACE(const char *format, ...) { #if defined(NVALGRIND) (void)format; return 0; #else /* NVALGRIND */ #if defined(_MSC_VER) || defined(__MINGW64__) uintptr_t _qzz_res; #else unsigned long _qzz_res; #endif va_list vargs; va_start(vargs, format); #if defined(_MSC_VER) || defined(__MINGW64__) _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF, (uintptr_t)format, (uintptr_t)&vargs, 0, 0, 0); #else _qzz_res = VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__PRINTF_BACKTRACE_VALIST_BY_REF, (unsigned long)format, (unsigned long)&vargs, 0, 0, 0); #endif va_end(vargs); return (int)_qzz_res; #endif /* NVALGRIND */ } /* These requests allow control to move from the simulated CPU to the real CPU, calling an arbitrary function. Note that the current ThreadId is inserted as the first argument. So this call: VALGRIND_NON_SIMD_CALL2(f, arg1, arg2) requires f to have this signature: Word f(Word tid, Word arg1, Word arg2) where "Word" is a word-sized type. Note that these client requests are not entirely reliable. For example, if you call a function with them that subsequently calls printf(), there's a high chance Valgrind will crash. Generally, your prospects of these working are made higher if the called function does not refer to any global variables, and does not refer to any libc or other functions (printf et al). Any kind of entanglement with libc or dynamic linking is likely to have a bad outcome, for tricky reasons which we've grappled with a lot in the past. */ #define VALGRIND_NON_SIMD_CALL0(_qyy_fn) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL0, \ _qyy_fn, \ 0, 0, 0, 0) #define VALGRIND_NON_SIMD_CALL1(_qyy_fn, _qyy_arg1) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL1, \ _qyy_fn, \ _qyy_arg1, 0, 0, 0) #define VALGRIND_NON_SIMD_CALL2(_qyy_fn, _qyy_arg1, _qyy_arg2) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL2, \ _qyy_fn, \ _qyy_arg1, _qyy_arg2, 0, 0) #define VALGRIND_NON_SIMD_CALL3(_qyy_fn, _qyy_arg1, _qyy_arg2, _qyy_arg3) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0 /* default return */, \ VG_USERREQ__CLIENT_CALL3, \ _qyy_fn, \ _qyy_arg1, _qyy_arg2, \ _qyy_arg3, 0) /* Counts the number of errors that have been recorded by a tool. Nb: the tool must record the errors with VG_(maybe_record_error)() or VG_(unique_error)() for them to be counted. */ #define VALGRIND_COUNT_ERRORS \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR( \ 0 /* default return */, \ VG_USERREQ__COUNT_ERRORS, \ 0, 0, 0, 0, 0) /* Several Valgrind tools (Memcheck, Massif, Helgrind, DRD) rely on knowing when heap blocks are allocated in order to give accurate results. This happens automatically for the standard allocator functions such as malloc(), calloc(), realloc(), memalign(), new, new[], free(), delete, delete[], etc. But if your program uses a custom allocator, this doesn't automatically happen, and Valgrind will not do as well. For example, if you allocate superblocks with mmap() and then allocates chunks of the superblocks, all Valgrind's observations will be at the mmap() level and it won't know that the chunks should be considered separate entities. In Memcheck's case, that means you probably won't get heap block overrun detection (because there won't be redzones marked as unaddressable) and you definitely won't get any leak detection. The following client requests allow a custom allocator to be annotated so that it can be handled accurately by Valgrind. VALGRIND_MALLOCLIKE_BLOCK marks a region of memory as having been allocated by a malloc()-like function. For Memcheck (an illustrative case), this does two things: - It records that the block has been allocated. This means any addresses within the block mentioned in error messages will be identified as belonging to the block. It also means that if the block isn't freed it will be detected by the leak checker. - It marks the block as being addressable and undefined (if 'is_zeroed' is not set), or addressable and defined (if 'is_zeroed' is set). This controls how accesses to the block by the program are handled. 'addr' is the start of the usable block (ie. after any redzone), 'sizeB' is its size. 'rzB' is the redzone size if the allocator can apply redzones -- these are blocks of padding at the start and end of each block. Adding redzones is recommended as it makes it much more likely Valgrind will spot block overruns. `is_zeroed' indicates if the memory is zeroed (or filled with another predictable value), as is the case for calloc(). VALGRIND_MALLOCLIKE_BLOCK should be put immediately after the point where a heap block -- that will be used by the client program -- is allocated. It's best to put it at the outermost level of the allocator if possible; for example, if you have a function my_alloc() which calls internal_alloc(), and the client request is put inside internal_alloc(), stack traces relating to the heap block will contain entries for both my_alloc() and internal_alloc(), which is probably not what you want. For Memcheck users: if you use VALGRIND_MALLOCLIKE_BLOCK to carve out custom blocks from within a heap block, B, that has been allocated with malloc/calloc/new/etc, then block B will be *ignored* during leak-checking -- the custom blocks will take precedence. VALGRIND_FREELIKE_BLOCK is the partner to VALGRIND_MALLOCLIKE_BLOCK. For Memcheck, it does two things: - It records that the block has been deallocated. This assumes that the block was annotated as having been allocated via VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued. - It marks the block as being unaddressable. VALGRIND_FREELIKE_BLOCK should be put immediately after the point where a heap block is deallocated. VALGRIND_RESIZEINPLACE_BLOCK informs a tool about reallocation. For Memcheck, it does four things: - It records that the size of a block has been changed. This assumes that the block was annotated as having been allocated via VALGRIND_MALLOCLIKE_BLOCK. Otherwise, an error will be issued. - If the block shrunk, it marks the freed memory as being unaddressable. - If the block grew, it marks the new area as undefined and defines a red zone past the end of the new block. - The V-bits of the overlap between the old and the new block are preserved. VALGRIND_RESIZEINPLACE_BLOCK should be put after allocation of the new block and before deallocation of the old block. In many cases, these three client requests will not be enough to get your allocator working well with Memcheck. More specifically, if your allocator writes to freed blocks in any way then a VALGRIND_MAKE_MEM_UNDEFINED call will be necessary to mark the memory as addressable just before the zeroing occurs, otherwise you'll get a lot of invalid write errors. For example, you'll need to do this if your allocator recycles freed blocks, but it zeroes them before handing them back out (via VALGRIND_MALLOCLIKE_BLOCK). Alternatively, if your allocator reuses freed blocks for allocator-internal data structures, VALGRIND_MAKE_MEM_UNDEFINED calls will also be necessary. Really, what's happening is a blurring of the lines between the client program and the allocator... after VALGRIND_FREELIKE_BLOCK is called, the memory should be considered unaddressable to the client program, but the allocator knows more than the rest of the client program and so may be able to safely access it. Extra client requests are necessary for Valgrind to understand the distinction between the allocator and the rest of the program. Ignored if addr == 0. */ #define VALGRIND_MALLOCLIKE_BLOCK(addr, sizeB, rzB, is_zeroed) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MALLOCLIKE_BLOCK, \ addr, sizeB, rzB, is_zeroed, 0) /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details. Ignored if addr == 0. */ #define VALGRIND_RESIZEINPLACE_BLOCK(addr, oldSizeB, newSizeB, rzB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__RESIZEINPLACE_BLOCK, \ addr, oldSizeB, newSizeB, rzB, 0) /* See the comment for VALGRIND_MALLOCLIKE_BLOCK for details. Ignored if addr == 0. */ #define VALGRIND_FREELIKE_BLOCK(addr, rzB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__FREELIKE_BLOCK, \ addr, rzB, 0, 0, 0) /* Create a memory pool. */ #define VALGRIND_CREATE_MEMPOOL(pool, rzB, is_zeroed) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL, \ pool, rzB, is_zeroed, 0, 0) /* Create a memory pool with some flags specifying extended behaviour. When flags is zero, the behaviour is identical to VALGRIND_CREATE_MEMPOOL. The flag VALGRIND_MEMPOOL_METAPOOL specifies that the pieces of memory associated with the pool using VALGRIND_MEMPOOL_ALLOC will be used by the application as superblocks to dole out MALLOC_LIKE blocks using VALGRIND_MALLOCLIKE_BLOCK. In other words, a meta pool is a "2 levels" pool : first level is the blocks described by VALGRIND_MEMPOOL_ALLOC. The second level blocks are described using VALGRIND_MALLOCLIKE_BLOCK. Note that the association between the pool and the second level blocks is implicit : second level blocks will be located inside first level blocks. It is necessary to use the VALGRIND_MEMPOOL_METAPOOL flag for such 2 levels pools, as otherwise valgrind will detect overlapping memory blocks, and will abort execution (e.g. during leak search). Such a meta pool can also be marked as an 'auto free' pool using the flag VALGRIND_MEMPOOL_AUTO_FREE, which must be OR-ed together with the VALGRIND_MEMPOOL_METAPOOL. For an 'auto free' pool, VALGRIND_MEMPOOL_FREE will automatically free the second level blocks that are contained inside the first level block freed with VALGRIND_MEMPOOL_FREE. In other words, calling VALGRIND_MEMPOOL_FREE will cause implicit calls to VALGRIND_FREELIKE_BLOCK for all the second level blocks included in the first level block. Note: it is an error to use the VALGRIND_MEMPOOL_AUTO_FREE flag without the VALGRIND_MEMPOOL_METAPOOL flag. */ #define VALGRIND_MEMPOOL_AUTO_FREE 1 #define VALGRIND_MEMPOOL_METAPOOL 2 #define VALGRIND_CREATE_MEMPOOL_EXT(pool, rzB, is_zeroed, flags) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CREATE_MEMPOOL, \ pool, rzB, is_zeroed, flags, 0) /* Destroy a memory pool. */ #define VALGRIND_DESTROY_MEMPOOL(pool) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__DESTROY_MEMPOOL, \ pool, 0, 0, 0, 0) /* Associate a piece of memory with a memory pool. */ #define VALGRIND_MEMPOOL_ALLOC(pool, addr, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_ALLOC, \ pool, addr, size, 0, 0) /* Disassociate a piece of memory from a memory pool. */ #define VALGRIND_MEMPOOL_FREE(pool, addr) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_FREE, \ pool, addr, 0, 0, 0) /* Disassociate any pieces outside a particular range. */ #define VALGRIND_MEMPOOL_TRIM(pool, addr, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_TRIM, \ pool, addr, size, 0, 0) /* Resize and/or move a piece associated with a memory pool. */ #define VALGRIND_MOVE_MEMPOOL(poolA, poolB) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MOVE_MEMPOOL, \ poolA, poolB, 0, 0, 0) /* Resize and/or move a piece associated with a memory pool. */ #define VALGRIND_MEMPOOL_CHANGE(pool, addrA, addrB, size) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__MEMPOOL_CHANGE, \ pool, addrA, addrB, size, 0) /* Return 1 if a mempool exists, else 0. */ #define VALGRIND_MEMPOOL_EXISTS(pool) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__MEMPOOL_EXISTS, \ pool, 0, 0, 0, 0) /* Mark a piece of memory as being a stack. Returns a stack id. start is the lowest addressable stack byte, end is the highest addressable stack byte. */ #define VALGRIND_STACK_REGISTER(start, end) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__STACK_REGISTER, \ start, end, 0, 0, 0) /* Unmark the piece of memory associated with a stack id as being a stack. */ #define VALGRIND_STACK_DEREGISTER(id) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_DEREGISTER, \ id, 0, 0, 0, 0) /* Change the start and end address of the stack id. start is the new lowest addressable stack byte, end is the new highest addressable stack byte. */ #define VALGRIND_STACK_CHANGE(id, start, end) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__STACK_CHANGE, \ id, start, end, 0, 0) /* Load PDB debug info for Wine PE image_map. */ #define VALGRIND_LOAD_PDB_DEBUGINFO(fd, ptr, total_size, delta) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__LOAD_PDB_DEBUGINFO, \ fd, ptr, total_size, delta, 0) /* Map a code address to a source file name and line number. buf64 must point to a 64-byte buffer in the caller's address space. The result will be dumped in there and is guaranteed to be zero terminated. If no info is found, the first byte is set to zero. */ #define VALGRIND_MAP_IP_TO_SRCLOC(addr, buf64) \ (unsigned)VALGRIND_DO_CLIENT_REQUEST_EXPR(0, \ VG_USERREQ__MAP_IP_TO_SRCLOC, \ addr, buf64, 0, 0, 0) /* Disable error reporting for this thread. Behaves in a stack like way, so you can safely call this multiple times provided that VALGRIND_ENABLE_ERROR_REPORTING is called the same number of times to re-enable reporting. The first call of this macro disables reporting. Subsequent calls have no effect except to increase the number of VALGRIND_ENABLE_ERROR_REPORTING calls needed to re-enable reporting. Child threads do not inherit this setting from their parents -- they are always created with reporting enabled. */ #define VALGRIND_DISABLE_ERROR_REPORTING \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \ 1, 0, 0, 0, 0) /* Re-enable error reporting, as per comments on VALGRIND_DISABLE_ERROR_REPORTING. */ #define VALGRIND_ENABLE_ERROR_REPORTING \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CHANGE_ERR_DISABLEMENT, \ -1, 0, 0, 0, 0) /* Execute a monitor command from the client program. If a connection is opened with GDB, the output will be sent according to the output mode set for vgdb. If no connection is opened, output will go to the log output. Returns 1 if command not recognised, 0 otherwise. */ #define VALGRIND_MONITOR_COMMAND(command) \ VALGRIND_DO_CLIENT_REQUEST_EXPR(0, VG_USERREQ__GDB_MONITOR_COMMAND, \ command, 0, 0, 0, 0) /* Change the value of a dynamic command line option. Note that unknown or not dynamically changeable options will cause a warning message to be output. */ #define VALGRIND_CLO_CHANGE(option) \ VALGRIND_DO_CLIENT_REQUEST_STMT(VG_USERREQ__CLO_CHANGE, \ option, 0, 0, 0, 0) #undef PLAT_x86_darwin #undef PLAT_amd64_darwin #undef PLAT_x86_win32 #undef PLAT_amd64_win64 #undef PLAT_x86_linux #undef PLAT_amd64_linux #undef PLAT_ppc32_linux #undef PLAT_ppc64be_linux #undef PLAT_ppc64le_linux #undef PLAT_arm_linux #undef PLAT_s390x_linux #undef PLAT_mips32_linux #undef PLAT_mips64_linux #undef PLAT_nanomips_linux #undef PLAT_x86_solaris #undef PLAT_amd64_solaris #endif /* __VALGRIND_H */ codspeed-4.4.1/instrument-hooks/includes/zig.h000064400000000000000000004662131046102023000175350ustar 00000000000000#undef linux #include #include #if defined(_MSC_VER) #define zig_msvc #elif defined(__clang__) #define zig_clang #define zig_gnuc #elif defined(__GNUC__) #define zig_gcc #define zig_gnuc #elif defined(__IBMC__) #define zig_xlc #elif defined(__TINYC__) #define zig_tinyc #elif defined(__slimcc__) #define zig_slimcc #endif #if defined(__aarch64__) || (defined(zig_msvc) && defined(_M_ARM64)) #define zig_aarch64 #elif defined(__thumb__) || (defined(zig_msvc) && defined(_M_ARM)) #define zig_thumb #define zig_arm #elif defined(__arm__) #define zig_arm #elif defined(__hexagon__) #define zig_hexagon #elif defined(__loongarch32) #define zig_loongarch32 #define zig_loongarch #elif defined(__loongarch64) #define zig_loongarch64 #define zig_loongarch #elif defined(__mips64) #define zig_mips64 #define zig_mips #elif defined(__mips__) #define zig_mips32 #define zig_mips #elif defined(__powerpc64__) #define zig_powerpc64 #define zig_powerpc #elif defined(__powerpc__) #define zig_powerpc32 #define zig_powerpc #elif defined(__riscv) && __riscv_xlen == 32 #define zig_riscv32 #define zig_riscv #elif defined(__riscv) && __riscv_xlen == 64 #define zig_riscv64 #define zig_riscv #elif defined(__s390x__) #define zig_s390x #elif defined(__sparc__) && defined(__arch64__) #define zig_sparc64 #define zig_sparc #elif defined(__sparc__) #define zig_sparc32 #define zig_sparc #elif defined(__wasm32__) #define zig_wasm32 #define zig_wasm #elif defined(__wasm64__) #define zig_wasm64 #define zig_wasm #elif defined(__i386__) || (defined(zig_msvc) && defined(_M_IX86)) #define zig_x86_32 #define zig_x86 #elif defined (__x86_64__) || (defined(zig_msvc) && defined(_M_X64)) #define zig_x86_64 #define zig_x86 #endif #if defined(zig_msvc) || __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ #define zig_little_endian 1 #define zig_big_endian 0 #else #define zig_little_endian 0 #define zig_big_endian 1 #endif #if defined(_AIX) #define zig_aix #elif defined(__MACH__) #define zig_darwin #elif defined(__DragonFly__) #define zig_dragonfly #define zig_bsd #elif defined(__EMSCRIPTEN__) #define zig_emscripten #elif defined(__FreeBSD__) #define zig_freebsd #define zig_bsd #elif defined(__Fuchsia__) #define zig_fuchsia #elif defined(__HAIKU__) #define zig_haiku #elif defined(__gnu_hurd__) #define zig_hurd #elif defined(__linux__) #define zig_linux #elif defined(__NetBSD__) #define zig_netbsd #define zig_bsd #elif defined(__OpenBSD__) #define zig_openbsd #define zig_bsd #elif defined(__SVR4) #define zig_solaris #elif defined(__wasi__) #define zig_wasi #elif defined(_WIN32) #define zig_windows #elif defined(__MVS__) #define zig_zos #endif #if defined(zig_windows) #define zig_coff #elif defined(__ELF__) #define zig_elf #elif defined(zig_zos) #define zig_goff #elif defined(zig_darwin) #define zig_macho #elif defined(zig_aix) #define zig_xcoff #endif #define zig_concat(lhs, rhs) lhs##rhs #define zig_expand_concat(lhs, rhs) zig_concat(lhs, rhs) #if defined(__has_include) #define zig_has_include(include) __has_include(include) #else #define zig_has_include(include) 0 #endif #if defined(__has_builtin) #define zig_has_builtin(builtin) __has_builtin(__builtin_##builtin) #else #define zig_has_builtin(builtin) 0 #endif #define zig_expand_has_builtin(b) zig_has_builtin(b) #if defined(__has_attribute) #define zig_has_attribute(attribute) __has_attribute(attribute) #else #define zig_has_attribute(attribute) 0 #endif #if __STDC_VERSION__ >= 202311L #define zig_threadlocal thread_local #elif __STDC_VERSION__ >= 201112L #define zig_threadlocal _Thread_local #elif defined(zig_gnuc) || defined(zig_slimcc) #define zig_threadlocal __thread #elif defined(zig_msvc) #define zig_threadlocal __declspec(thread) #else #define zig_threadlocal zig_threadlocal_unavailable #endif #if defined(zig_msvc) #define zig_const_arr #define zig_callconv(c) __##c #else #define zig_const_arr static const #define zig_callconv(c) __attribute__((c)) #endif #if zig_has_attribute(naked) || defined(zig_gcc) #define zig_naked_decl __attribute__((naked)) #define zig_naked __attribute__((naked)) #elif defined(zig_msvc) #define zig_naked_decl #define zig_naked __declspec(naked) #else #define zig_naked_decl zig_naked_unavailable #define zig_naked zig_naked_unavailable #endif #if zig_has_attribute(cold) #define zig_cold __attribute__((cold)) #else #define zig_cold #endif #if zig_has_attribute(flatten) #define zig_maybe_flatten __attribute__((flatten)) #else #define zig_maybe_flatten #endif #if zig_has_attribute(noinline) #define zig_never_inline __attribute__((noinline)) zig_maybe_flatten #elif defined(zig_msvc) #define zig_never_inline __declspec(noinline) zig_maybe_flatten #else #define zig_never_inline zig_never_inline_unavailable #endif #if zig_has_attribute(not_tail_called) #define zig_never_tail __attribute__((not_tail_called)) zig_never_inline #else #define zig_never_tail zig_never_tail_unavailable #endif #if zig_has_attribute(musttail) #define zig_always_tail __attribute__((musttail)) #else #define zig_always_tail zig_always_tail_unavailable #endif #if __STDC_VERSION__ >= 199901L #define zig_restrict restrict #elif defined(zig_gnuc) || defined(zig_tinyc) #define zig_restrict __restrict #else #define zig_restrict #endif #if zig_has_attribute(no_builtin) #define zig_no_builtin __attribute__((no_builtin)) #else #define zig_no_builtin #endif #if zig_has_attribute(aligned) || defined(zig_tinyc) #define zig_under_align(alignment) __attribute__((aligned(alignment))) #elif defined(zig_msvc) #define zig_under_align(alignment) __declspec(align(alignment)) #else #define zig_under_align zig_align_unavailable #endif #if __STDC_VERSION__ >= 202311L #define zig_align(alignment) alignas(alignment) #elif __STDC_VERSION__ >= 201112L #define zig_align(alignment) _Alignas(alignment) #else #define zig_align(alignment) zig_under_align(alignment) #endif #if zig_has_attribute(aligned) || defined(zig_tinyc) #define zig_align_fn(alignment) __attribute__((aligned(alignment))) #elif defined(zig_msvc) #define zig_align_fn(alignment) #else #define zig_align_fn zig_align_fn_unavailable #endif #if zig_has_attribute(packed) || defined(zig_tinyc) #define zig_packed(definition) __attribute__((packed)) definition #elif defined(zig_msvc) #define zig_packed(definition) __pragma(pack(1)) definition __pragma(pack()) #else #define zig_packed(definition) zig_packed_unavailable #endif #if zig_has_attribute(section) || defined(zig_tinyc) #define zig_linksection(name) __attribute__((section(name))) #define zig_linksection_fn zig_linksection #elif defined(zig_msvc) #define zig_linksection(name) __pragma(section(name, read, write)) __declspec(allocate(name)) #define zig_linksection_fn(name) __pragma(section(name, read, execute)) __declspec(code_seg(name)) #else #define zig_linksection(name) zig_linksection_unavailable #define zig_linksection_fn zig_linksection #endif #if zig_has_builtin(unreachable) || defined(zig_gcc) || defined(zig_tinyc) #define zig_unreachable() __builtin_unreachable() #elif defined(zig_msvc) #define zig_unreachable() __assume(0) #else #define zig_unreachable() #endif #if defined(__cplusplus) #define zig_extern extern "C" #else #define zig_extern extern #endif #if defined(zig_msvc) #if defined(zig_x86_64) #define zig_mangle_c(symbol) symbol #else /* zig_x86_64 */ #define zig_mangle_c(symbol) "_" symbol #endif /* zig_x86_64 */ #else /* zig_msvc */ #if defined(zig_macho) #define zig_mangle_c(symbol) "_" symbol #else /* zig_macho */ #define zig_mangle_c(symbol) symbol #endif /* zig_macho */ #endif /* zig_msvc */ #if defined(zig_msvc) #define zig_export(symbol, name) ; \ __pragma(comment(linker, "/alternatename:" zig_mangle_c(name) "=" zig_mangle_c(symbol))) #elif (zig_has_attribute(alias) || defined(zig_tinyc)) && !defined(zig_macho) #define zig_export(symbol, name) __attribute__((alias(symbol))) #else #define zig_export(symbol, name) ; \ __asm(zig_mangle_c(name) " = " zig_mangle_c(symbol)) #endif #define zig_mangled_tentative zig_mangled #define zig_mangled_final zig_mangled #if defined(zig_msvc) #define zig_mangled(mangled, unmangled) ; \ zig_export(#mangled, unmangled) #define zig_mangled_export(mangled, unmangled, symbol) \ zig_export(unmangled, #mangled) \ zig_export(symbol, unmangled) #else /* zig_msvc */ #define zig_mangled(mangled, unmangled) __asm(zig_mangle_c(unmangled)) #define zig_mangled_export(mangled, unmangled, symbol) \ zig_mangled_final(mangled, unmangled) \ zig_export(symbol, unmangled) #endif /* zig_msvc */ #if defined(zig_msvc) #define zig_import(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type fn_name sig_args;\ __pragma(comment(linker, "/alternatename:" zig_mangle_c(#fn_name) "=" zig_mangle_c(#libc_name))); #define zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) zig_import(Type, fn_name, libc_name, sig_args, call_args) #else /* zig_msvc */ #define zig_import(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type fn_name sig_args __asm(zig_mangle_c(#libc_name)); #define zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) zig_extern Type libc_name sig_args; \ static inline Type fn_name sig_args { return libc_name call_args; } #endif #define zig_expand_import_0(Type, fn_name, libc_name, sig_args, call_args) zig_import(Type, fn_name, libc_name, sig_args, call_args) #define zig_expand_import_1(Type, fn_name, libc_name, sig_args, call_args) zig_import_builtin(Type, fn_name, libc_name, sig_args, call_args) #if zig_has_attribute(weak) || defined(zig_gcc) || defined(zig_tinyc) #define zig_weak_linkage __attribute__((weak)) #define zig_weak_linkage_fn __attribute__((weak)) #elif defined(zig_msvc) #define zig_weak_linkage __declspec(selectany) #define zig_weak_linkage_fn #else #define zig_weak_linkage zig_weak_linkage_unavailable #define zig_weak_linkage_fn zig_weak_linkage_unavailable #endif #if defined(zig_gnuc) || defined(zig_tinyc) || defined(zig_slimcc) #define zig_gnuc_asm #endif #if zig_has_builtin(trap) #define zig_trap() __builtin_trap() #elif defined(zig_msvc) #if defined(zig_x86) #define zig_trap() __ud2() #else #define zig_trap() __fastfail(7) #endif #elif defined(zig_gnuc_asm) #if defined(zig_thumb) #define zig_trap() __asm__ volatile("udf #0xfe") #elif defined(zig_arm) || defined(zig_aarch64) #define zig_trap() __asm__ volatile("udf #0xfdee") #elif defined(zig_hexagon) #define zig_trap() __asm__ volatile("r27:26 = memd(#0xbadc0fee)") #elif defined(zig_loongarch) || defined(zig_powerpc) #define zig_trap() __asm__ volatile(".word 0x0") #elif defined(zig_mips) #define zig_trap() __asm__ volatile(".word 0x3d") #elif defined(zig_riscv) #define zig_trap() __asm__ volatile("unimp") #elif defined(zig_s390x) #define zig_trap() __asm__ volatile("j 0x2") #elif defined(zig_sparc) #define zig_trap() __asm__ volatile("illtrap") #elif defined(zig_x86) #define zig_trap() __asm__ volatile("ud2") #else #define zig_trap() zig_trap_unavailable #endif #else #define zig_trap() zig_trap_unavailable #endif #if zig_has_builtin(debugtrap) #define zig_breakpoint() __builtin_debugtrap() #elif defined(zig_msvc) #define zig_breakpoint() __debugbreak() #elif defined(zig_gnuc_asm) #if defined(zig_arm) #define zig_breakpoint() __asm__ volatile("bkpt #0x0") #elif defined(zig_aarch64) #define zig_breakpoint() __asm__ volatile("brk #0xf000") #elif defined(zig_hexagon) #define zig_breakpoint() __asm__ volatile("brkpt") #elif defined(zig_loongarch) #define zig_breakpoint() __asm__ volatile("break 0x0") #elif defined(zig_mips) #define zig_breakpoint() __asm__ volatile("break") #elif defined(zig_powerpc) #define zig_breakpoint() __asm__ volatile("trap") #elif defined(zig_riscv) #define zig_breakpoint() __asm__ volatile("ebreak") #elif defined(zig_s390x) #define zig_breakpoint() __asm__ volatile("j 0x6") #elif defined(zig_sparc) #define zig_breakpoint() __asm__ volatile("ta 0x1") #elif defined(zig_x86) #define zig_breakpoint() __asm__ volatile("int $0x3") #else #define zig_breakpoint() zig_breakpoint_unavailable #endif #else #define zig_breakpoint() zig_breakpoint_unavailable #endif #if zig_has_builtin(return_address) || defined(zig_gcc) || defined(zig_tinyc) #define zig_return_address() __builtin_extract_return_addr(__builtin_return_address(0)) #elif defined(zig_msvc) #define zig_return_address() _ReturnAddress() #else #define zig_return_address() 0 #endif #if zig_has_builtin(frame_address) || defined(zig_gcc) || defined(zig_tinyc) #define zig_frame_address() __builtin_frame_address(0) #elif defined(zig_msvc) #define zig_frame_address() _AddressOfReturnAddress() #else #define zig_frame_address() 0 #endif #if zig_has_builtin(prefetch) || defined(zig_gcc) #define zig_prefetch(addr, rw, locality) __builtin_prefetch(addr, rw, locality) #else #define zig_prefetch(addr, rw, locality) #endif #if zig_has_builtin(memory_size) && zig_has_builtin(memory_grow) #define zig_wasm_memory_size(index) __builtin_wasm_memory_size(index) #define zig_wasm_memory_grow(index, delta) __builtin_wasm_memory_grow(index, delta) #else #define zig_wasm_memory_size(index) zig_unimplemented() #define zig_wasm_memory_grow(index, delta) zig_unimplemented() #endif #if __STDC_VERSION__ >= 202311L #define zig_noreturn [[noreturn]] #elif __STDC_VERSION__ >= 201112L #define zig_noreturn _Noreturn #elif zig_has_attribute(noreturn) || defined(zig_gcc) || defined(zig_tinyc) #define zig_noreturn __attribute__((noreturn)) #elif defined(zig_msvc) #define zig_noreturn __declspec(noreturn) #else #define zig_noreturn #endif #define zig_compiler_rt_abbrev_uint32_t si #define zig_compiler_rt_abbrev_int32_t si #define zig_compiler_rt_abbrev_uint64_t di #define zig_compiler_rt_abbrev_int64_t di #define zig_compiler_rt_abbrev_zig_u128 ti #define zig_compiler_rt_abbrev_zig_i128 ti #define zig_compiler_rt_abbrev_zig_f16 hf #define zig_compiler_rt_abbrev_zig_f32 sf #define zig_compiler_rt_abbrev_zig_f64 df #define zig_compiler_rt_abbrev_zig_f80 xf #define zig_compiler_rt_abbrev_zig_f128 tf zig_extern void *memcpy (void *zig_restrict, void const *zig_restrict, size_t); zig_extern void *memset (void *, int, size_t); /* ================ Bool and 8/16/32/64-bit Integer Support ================= */ #include #define zig_bitSizeOf(T) (CHAR_BIT * sizeof(T)) #if __STDC_VERSION__ >= 202311L /* bool, true, and false are provided by the language. */ #elif __STDC_VERSION__ >= 199901L || zig_has_include() #include #else typedef char bool; #define false 0 #define true 1 #endif #if __STDC_VERSION__ >= 199901L || defined(zig_msvc) || zig_has_include() #include #else #if SCHAR_MIN == ~0x7F && SCHAR_MAX == 0x7F && UCHAR_MAX == 0xFF typedef unsigned char uint8_t; typedef signed char int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif SHRT_MIN == ~0x7F && SHRT_MAX == 0x7F && USHRT_MAX == 0xFF typedef unsigned short uint8_t; typedef signed short int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif INT_MIN == ~0x7F && INT_MAX == 0x7F && UINT_MAX == 0xFF typedef unsigned int uint8_t; typedef signed int int8_t; #define INT8_C(c) c #define UINT8_C(c) c##U #elif LONG_MIN == ~0x7F && LONG_MAX == 0x7F && ULONG_MAX == 0xFF typedef unsigned long uint8_t; typedef signed long int8_t; #define INT8_C(c) c##L #define UINT8_C(c) c##LU #elif LLONG_MIN == ~0x7F && LLONG_MAX == 0x7F && ULLONG_MAX == 0xFF typedef unsigned long long uint8_t; typedef signed long long int8_t; #define INT8_C(c) c##LL #define UINT8_C(c) c##LLU #endif #define INT8_MIN (~INT8_C(0x7F)) #define INT8_MAX ( INT8_C(0x7F)) #define UINT8_MAX ( INT8_C(0xFF)) #if SCHAR_MIN == ~0x7FFF && SCHAR_MAX == 0x7FFF && UCHAR_MAX == 0xFFFF typedef unsigned char uint16_t; typedef signed char int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif SHRT_MIN == ~0x7FFF && SHRT_MAX == 0x7FFF && USHRT_MAX == 0xFFFF typedef unsigned short uint16_t; typedef signed short int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif INT_MIN == ~0x7FFF && INT_MAX == 0x7FFF && UINT_MAX == 0xFFFF typedef unsigned int uint16_t; typedef signed int int16_t; #define INT16_C(c) c #define UINT16_C(c) c##U #elif LONG_MIN == ~0x7FFF && LONG_MAX == 0x7FFF && ULONG_MAX == 0xFFFF typedef unsigned long uint16_t; typedef signed long int16_t; #define INT16_C(c) c##L #define UINT16_C(c) c##LU #elif LLONG_MIN == ~0x7FFF && LLONG_MAX == 0x7FFF && ULLONG_MAX == 0xFFFF typedef unsigned long long uint16_t; typedef signed long long int16_t; #define INT16_C(c) c##LL #define UINT16_C(c) c##LLU #endif #define INT16_MIN (~INT16_C(0x7FFF)) #define INT16_MAX ( INT16_C(0x7FFF)) #define UINT16_MAX ( INT16_C(0xFFFF)) #if SCHAR_MIN == ~0x7FFFFFFF && SCHAR_MAX == 0x7FFFFFFF && UCHAR_MAX == 0xFFFFFFFF typedef unsigned char uint32_t; typedef signed char int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif SHRT_MIN == ~0x7FFFFFFF && SHRT_MAX == 0x7FFFFFFF && USHRT_MAX == 0xFFFFFFFF typedef unsigned short uint32_t; typedef signed short int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif INT_MIN == ~0x7FFFFFFF && INT_MAX == 0x7FFFFFFF && UINT_MAX == 0xFFFFFFFF typedef unsigned int uint32_t; typedef signed int int32_t; #define INT32_C(c) c #define UINT32_C(c) c##U #elif LONG_MIN == ~0x7FFFFFFF && LONG_MAX == 0x7FFFFFFF && ULONG_MAX == 0xFFFFFFFF typedef unsigned long uint32_t; typedef signed long int32_t; #define INT32_C(c) c##L #define UINT32_C(c) c##LU #elif LLONG_MIN == ~0x7FFFFFFF && LLONG_MAX == 0x7FFFFFFF && ULLONG_MAX == 0xFFFFFFFF typedef unsigned long long uint32_t; typedef signed long long int32_t; #define INT32_C(c) c##LL #define UINT32_C(c) c##LLU #endif #define INT32_MIN (~INT32_C(0x7FFFFFFF)) #define INT32_MAX ( INT32_C(0x7FFFFFFF)) #define UINT32_MAX ( INT32_C(0xFFFFFFFF)) #if SCHAR_MIN == ~0x7FFFFFFFFFFFFFFF && SCHAR_MAX == 0x7FFFFFFFFFFFFFFF && UCHAR_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned char uint64_t; typedef signed char int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif SHRT_MIN == ~0x7FFFFFFFFFFFFFFF && SHRT_MAX == 0x7FFFFFFFFFFFFFFF && USHRT_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned short uint64_t; typedef signed short int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif INT_MIN == ~0x7FFFFFFFFFFFFFFF && INT_MAX == 0x7FFFFFFFFFFFFFFF && UINT_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned int uint64_t; typedef signed int int64_t; #define INT64_C(c) c #define UINT64_C(c) c##U #elif LONG_MIN == ~0x7FFFFFFFFFFFFFFF && LONG_MAX == 0x7FFFFFFFFFFFFFFF && ULONG_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned long uint64_t; typedef signed long int64_t; #define INT64_C(c) c##L #define UINT64_C(c) c##LU #elif LLONG_MIN == ~0x7FFFFFFFFFFFFFFF && LLONG_MAX == 0x7FFFFFFFFFFFFFFF && ULLONG_MAX == 0xFFFFFFFFFFFFFFFF typedef unsigned long long uint64_t; typedef signed long long int64_t; #define INT64_C(c) c##LL #define UINT64_C(c) c##LLU #endif #define INT64_MIN (~INT64_C(0x7FFFFFFFFFFFFFFF)) #define INT64_MAX ( INT64_C(0x7FFFFFFFFFFFFFFF)) #define UINT64_MAX ( INT64_C(0xFFFFFFFFFFFFFFFF)) typedef size_t uintptr_t; typedef ptrdiff_t intptr_t; #endif #define zig_minInt_i8 INT8_MIN #define zig_maxInt_i8 INT8_MAX #define zig_minInt_u8 UINT8_C(0) #define zig_maxInt_u8 UINT8_MAX #define zig_minInt_i16 INT16_MIN #define zig_maxInt_i16 INT16_MAX #define zig_minInt_u16 UINT16_C(0) #define zig_maxInt_u16 UINT16_MAX #define zig_minInt_i32 INT32_MIN #define zig_maxInt_i32 INT32_MAX #define zig_minInt_u32 UINT32_C(0) #define zig_maxInt_u32 UINT32_MAX #define zig_minInt_i64 INT64_MIN #define zig_maxInt_i64 INT64_MAX #define zig_minInt_u64 UINT64_C(0) #define zig_maxInt_u64 UINT64_MAX #define zig_intLimit(s, w, limit, bits) zig_shr_##s##w(zig_##limit##Int_##s##w, w - (bits)) #define zig_minInt_i(w, bits) zig_intLimit(i, w, min, bits) #define zig_maxInt_i(w, bits) zig_intLimit(i, w, max, bits) #define zig_minInt_u(w, bits) zig_intLimit(u, w, min, bits) #define zig_maxInt_u(w, bits) zig_intLimit(u, w, max, bits) #define zig_operator(Type, RhsType, operation, operator) \ static inline Type zig_##operation(Type lhs, RhsType rhs) { \ return lhs operator rhs; \ } #define zig_basic_operator(Type, operation, operator) \ zig_operator(Type, Type, operation, operator) #define zig_shift_operator(Type, operation, operator) \ zig_operator(Type, uint8_t, operation, operator) #define zig_int_helpers(w, PromotedUnsigned) \ zig_basic_operator(uint##w##_t, and_u##w, &) \ zig_basic_operator( int##w##_t, and_i##w, &) \ zig_basic_operator(uint##w##_t, or_u##w, |) \ zig_basic_operator( int##w##_t, or_i##w, |) \ zig_basic_operator(uint##w##_t, xor_u##w, ^) \ zig_basic_operator( int##w##_t, xor_i##w, ^) \ zig_shift_operator(uint##w##_t, shl_u##w, <<) \ zig_shift_operator( int##w##_t, shl_i##w, <<) \ zig_shift_operator(uint##w##_t, shr_u##w, >>) \ \ static inline int##w##_t zig_shr_i##w(int##w##_t lhs, uint8_t rhs) { \ int##w##_t sign_mask = lhs < INT##w##_C(0) ? -INT##w##_C(1) : INT##w##_C(0); \ return ((lhs ^ sign_mask) >> rhs) ^ sign_mask; \ } \ \ static inline uint##w##_t zig_not_u##w(uint##w##_t val, uint8_t bits) { \ return val ^ zig_maxInt_u(w, bits); \ } \ \ static inline int##w##_t zig_not_i##w(int##w##_t val, uint8_t bits) { \ (void)bits; \ return ~val; \ } \ \ static inline uint##w##_t zig_wrap_u##w(uint##w##_t val, uint8_t bits) { \ return val & zig_maxInt_u(w, bits); \ } \ \ static inline int##w##_t zig_wrap_i##w(int##w##_t val, uint8_t bits) { \ return (val & UINT##w##_C(1) << (bits - UINT8_C(1))) != 0 \ ? val | zig_minInt_i(w, bits) : val & zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_abs_i##w(int##w##_t val) { \ return (val < 0) ? -(uint##w##_t)val : (uint##w##_t)val; \ } \ \ zig_basic_operator(uint##w##_t, div_floor_u##w, /) \ \ static inline int##w##_t zig_div_floor_i##w(int##w##_t lhs, int##w##_t rhs) { \ return lhs / rhs + (lhs % rhs != INT##w##_C(0) ? zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ } \ \ zig_basic_operator(uint##w##_t, mod_u##w, %) \ \ static inline int##w##_t zig_mod_i##w(int##w##_t lhs, int##w##_t rhs) { \ int##w##_t rem = lhs % rhs; \ return rem + (rem != INT##w##_C(0) ? rhs & zig_shr_i##w(lhs ^ rhs, UINT8_C(w) - UINT8_C(1)) : INT##w##_C(0)); \ } \ \ static inline uint##w##_t zig_shlw_u##w(uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \ return zig_wrap_u##w(zig_shl_u##w(lhs, rhs), bits); \ } \ \ static inline int##w##_t zig_shlw_i##w(int##w##_t lhs, uint8_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)zig_shl_u##w((uint##w##_t)lhs, rhs), bits); \ } \ \ static inline uint##w##_t zig_addw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w(lhs + rhs, bits); \ } \ \ static inline int##w##_t zig_addw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs + (uint##w##_t)rhs), bits); \ } \ \ static inline uint##w##_t zig_subw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w(lhs - rhs, bits); \ } \ \ static inline int##w##_t zig_subw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs - (uint##w##_t)rhs), bits); \ } \ \ static inline uint##w##_t zig_mulw_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ return zig_wrap_u##w((PromotedUnsigned)lhs * rhs, bits); \ } \ \ static inline int##w##_t zig_mulw_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ return zig_wrap_i##w((int##w##_t)((uint##w##_t)lhs * (uint##w##_t)rhs), bits); \ } #if UINT8_MAX <= UINT_MAX zig_int_helpers(8, unsigned int) #elif UINT8_MAX <= ULONG_MAX zig_int_helpers(8, unsigned long) #elif UINT8_MAX <= ULLONG_MAX zig_int_helpers(8, unsigned long long) #else zig_int_helpers(8, uint8_t) #endif #if UINT16_MAX <= UINT_MAX zig_int_helpers(16, unsigned int) #elif UINT16_MAX <= ULONG_MAX zig_int_helpers(16, unsigned long) #elif UINT16_MAX <= ULLONG_MAX zig_int_helpers(16, unsigned long long) #else zig_int_helpers(16, uint16_t) #endif #if UINT32_MAX <= UINT_MAX zig_int_helpers(32, unsigned int) #elif UINT32_MAX <= ULONG_MAX zig_int_helpers(32, unsigned long) #elif UINT32_MAX <= ULLONG_MAX zig_int_helpers(32, unsigned long long) #else zig_int_helpers(32, uint32_t) #endif #if UINT64_MAX <= UINT_MAX zig_int_helpers(64, unsigned int) #elif UINT64_MAX <= ULONG_MAX zig_int_helpers(64, unsigned long) #elif UINT64_MAX <= ULLONG_MAX zig_int_helpers(64, unsigned long long) #else zig_int_helpers(64, uint64_t) #endif static inline bool zig_addo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_addw_u32(lhs, rhs, bits); return *res < lhs; #endif } zig_extern int32_t __addosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_addo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __addosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_addo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_addw_u64(lhs, rhs, bits); return *res < lhs; #endif } zig_extern int64_t __addodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_addo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __addodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_addo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_addo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_addo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_addo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_addo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_addo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_addo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_addo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } static inline bool zig_subo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_subw_u32(lhs, rhs, bits); return *res > lhs; #endif } zig_extern int32_t __subosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_subo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __subosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_subo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_subw_u64(lhs, rhs, bits); return *res > lhs; #endif } zig_extern int64_t __subodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_subo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __subodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_subo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_subo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_subo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_subo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_subo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_subo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_subo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_subo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } static inline bool zig_mulo_u32(uint32_t *res, uint32_t lhs, uint32_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint32_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u32(full_res, bits); return overflow || full_res < zig_minInt_u(32, bits) || full_res > zig_maxInt_u(32, bits); #else *res = zig_mulw_u32(lhs, rhs, bits); return rhs != UINT32_C(0) && lhs > zig_maxInt_u(32, bits) / rhs; #endif } zig_extern int32_t __mulosi4(int32_t lhs, int32_t rhs, int *overflow); static inline bool zig_mulo_i32(int32_t *res, int32_t lhs, int32_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int32_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; int32_t full_res = __mulosi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i32(full_res, bits); return overflow || full_res < zig_minInt_i(32, bits) || full_res > zig_maxInt_i(32, bits); } static inline bool zig_mulo_u64(uint64_t *res, uint64_t lhs, uint64_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint64_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u64(full_res, bits); return overflow || full_res < zig_minInt_u(64, bits) || full_res > zig_maxInt_u(64, bits); #else *res = zig_mulw_u64(lhs, rhs, bits); return rhs != UINT64_C(0) && lhs > zig_maxInt_u(64, bits) / rhs; #endif } zig_extern int64_t __mulodi4(int64_t lhs, int64_t rhs, int *overflow); static inline bool zig_mulo_i64(int64_t *res, int64_t lhs, int64_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int64_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; int64_t full_res = __mulodi4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i64(full_res, bits); return overflow || full_res < zig_minInt_i(64, bits) || full_res > zig_maxInt_i(64, bits); } static inline bool zig_mulo_u8(uint8_t *res, uint8_t lhs, uint8_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint8_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u8(full_res, bits); return overflow || full_res < zig_minInt_u(8, bits) || full_res > zig_maxInt_u(8, bits); #else uint32_t full_res; bool overflow = zig_mulo_u32(&full_res, lhs, rhs, bits); *res = (uint8_t)full_res; return overflow; #endif } static inline bool zig_mulo_i8(int8_t *res, int8_t lhs, int8_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int8_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_i8(full_res, bits); return overflow || full_res < zig_minInt_i(8, bits) || full_res > zig_maxInt_i(8, bits); #else int32_t full_res; bool overflow = zig_mulo_i32(&full_res, lhs, rhs, bits); *res = (int8_t)full_res; return overflow; #endif } static inline bool zig_mulo_u16(uint16_t *res, uint16_t lhs, uint16_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) uint16_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u16(full_res, bits); return overflow || full_res < zig_minInt_u(16, bits) || full_res > zig_maxInt_u(16, bits); #else uint32_t full_res; bool overflow = zig_mulo_u32(&full_res, lhs, rhs, bits); *res = (uint16_t)full_res; return overflow; #endif } static inline bool zig_mulo_i16(int16_t *res, int16_t lhs, int16_t rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) || defined(zig_gcc) int16_t full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_i16(full_res, bits); return overflow || full_res < zig_minInt_i(16, bits) || full_res > zig_maxInt_i(16, bits); #else int32_t full_res; bool overflow = zig_mulo_i32(&full_res, lhs, rhs, bits); *res = (int16_t)full_res; return overflow; #endif } #define zig_int_builtins(w) \ static inline bool zig_shlo_u##w(uint##w##_t *res, uint##w##_t lhs, uint8_t rhs, uint8_t bits) { \ *res = zig_shlw_u##w(lhs, rhs, bits); \ return lhs > zig_maxInt_u(w, bits) >> rhs; \ } \ \ static inline bool zig_shlo_i##w(int##w##_t *res, int##w##_t lhs, uint8_t rhs, uint8_t bits) { \ *res = zig_shlw_i##w(lhs, rhs, bits); \ int##w##_t mask = (int##w##_t)(UINT##w##_MAX << (bits - rhs - 1)); \ return (lhs & mask) != INT##w##_C(0) && (lhs & mask) != mask; \ } \ \ static inline uint##w##_t zig_shls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ if (rhs >= bits) return lhs != UINT##w##_C(0) ? zig_maxInt_u(w, bits) : lhs; \ return zig_shlo_u##w(&res, lhs, (uint8_t)rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_shls_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if ((uint##w##_t)rhs < (uint##w##_t)bits && !zig_shlo_i##w(&res, lhs, (uint8_t)rhs, bits)) return res; \ return lhs < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_adds_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_addo_u##w(&res, lhs, rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_adds_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_addo_i##w(&res, lhs, rhs, bits)) return res; \ return res >= INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_subs_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_subo_u##w(&res, lhs, rhs, bits) ? zig_minInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_subs_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_subo_i##w(&res, lhs, rhs, bits)) return res; \ return res >= INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } \ \ static inline uint##w##_t zig_muls_u##w(uint##w##_t lhs, uint##w##_t rhs, uint8_t bits) { \ uint##w##_t res; \ return zig_mulo_u##w(&res, lhs, rhs, bits) ? zig_maxInt_u(w, bits) : res; \ } \ \ static inline int##w##_t zig_muls_i##w(int##w##_t lhs, int##w##_t rhs, uint8_t bits) { \ int##w##_t res; \ if (!zig_mulo_i##w(&res, lhs, rhs, bits)) return res; \ return (lhs ^ rhs) < INT##w##_C(0) ? zig_minInt_i(w, bits) : zig_maxInt_i(w, bits); \ } zig_int_builtins(8) zig_int_builtins(16) zig_int_builtins(32) zig_int_builtins(64) #define zig_builtin8(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin8; #define zig_builtin16(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin16; #if INT_MIN <= INT32_MIN #define zig_builtin32(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin32; #elif LONG_MIN <= INT32_MIN #define zig_builtin32(name, val) __builtin_##name##l(val) typedef unsigned long zig_Builtin32; #endif #if INT_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name(val) typedef unsigned int zig_Builtin64; #elif LONG_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name##l(val) typedef unsigned long zig_Builtin64; #elif LLONG_MIN <= INT64_MIN #define zig_builtin64(name, val) __builtin_##name##ll(val) typedef unsigned long long zig_Builtin64; #endif static inline uint8_t zig_byte_swap_u8(uint8_t val, uint8_t bits) { return zig_wrap_u8(val >> (8 - bits), bits); } static inline int8_t zig_byte_swap_i8(int8_t val, uint8_t bits) { return zig_wrap_i8((int8_t)zig_byte_swap_u8((uint8_t)val, bits), bits); } static inline uint16_t zig_byte_swap_u16(uint16_t val, uint8_t bits) { uint16_t full_res; #if zig_has_builtin(bswap16) || defined(zig_gcc) full_res = __builtin_bswap16(val); #else full_res = (uint16_t)zig_byte_swap_u8((uint8_t)(val >> 0), 8) << 8 | (uint16_t)zig_byte_swap_u8((uint8_t)(val >> 8), 8) >> 0; #endif return zig_wrap_u16(full_res >> (16 - bits), bits); } static inline int16_t zig_byte_swap_i16(int16_t val, uint8_t bits) { return zig_wrap_i16((int16_t)zig_byte_swap_u16((uint16_t)val, bits), bits); } static inline uint32_t zig_byte_swap_u32(uint32_t val, uint8_t bits) { uint32_t full_res; #if zig_has_builtin(bswap32) || defined(zig_gcc) full_res = __builtin_bswap32(val); #else full_res = (uint32_t)zig_byte_swap_u16((uint16_t)(val >> 0), 16) << 16 | (uint32_t)zig_byte_swap_u16((uint16_t)(val >> 16), 16) >> 0; #endif return zig_wrap_u32(full_res >> (32 - bits), bits); } static inline int32_t zig_byte_swap_i32(int32_t val, uint8_t bits) { return zig_wrap_i32((int32_t)zig_byte_swap_u32((uint32_t)val, bits), bits); } static inline uint64_t zig_byte_swap_u64(uint64_t val, uint8_t bits) { uint64_t full_res; #if zig_has_builtin(bswap64) || defined(zig_gcc) full_res = __builtin_bswap64(val); #else full_res = (uint64_t)zig_byte_swap_u32((uint32_t)(val >> 0), 32) << 32 | (uint64_t)zig_byte_swap_u32((uint32_t)(val >> 32), 32) >> 0; #endif return zig_wrap_u64(full_res >> (64 - bits), bits); } static inline int64_t zig_byte_swap_i64(int64_t val, uint8_t bits) { return zig_wrap_i64((int64_t)zig_byte_swap_u64((uint64_t)val, bits), bits); } static inline uint8_t zig_bit_reverse_u8(uint8_t val, uint8_t bits) { uint8_t full_res; #if zig_has_builtin(bitreverse8) full_res = __builtin_bitreverse8(val); #else static uint8_t const lut[0x10] = { 0x0, 0x8, 0x4, 0xc, 0x2, 0xa, 0x6, 0xe, 0x1, 0x9, 0x5, 0xd, 0x3, 0xb, 0x7, 0xf }; full_res = lut[val >> 0 & 0xF] << 4 | lut[val >> 4 & 0xF] << 0; #endif return zig_wrap_u8(full_res >> (8 - bits), bits); } static inline int8_t zig_bit_reverse_i8(int8_t val, uint8_t bits) { return zig_wrap_i8((int8_t)zig_bit_reverse_u8((uint8_t)val, bits), bits); } static inline uint16_t zig_bit_reverse_u16(uint16_t val, uint8_t bits) { uint16_t full_res; #if zig_has_builtin(bitreverse16) full_res = __builtin_bitreverse16(val); #else full_res = (uint16_t)zig_bit_reverse_u8((uint8_t)(val >> 0), 8) << 8 | (uint16_t)zig_bit_reverse_u8((uint8_t)(val >> 8), 8) >> 0; #endif return zig_wrap_u16(full_res >> (16 - bits), bits); } static inline int16_t zig_bit_reverse_i16(int16_t val, uint8_t bits) { return zig_wrap_i16((int16_t)zig_bit_reverse_u16((uint16_t)val, bits), bits); } static inline uint32_t zig_bit_reverse_u32(uint32_t val, uint8_t bits) { uint32_t full_res; #if zig_has_builtin(bitreverse32) full_res = __builtin_bitreverse32(val); #else full_res = (uint32_t)zig_bit_reverse_u16((uint16_t)(val >> 0), 16) << 16 | (uint32_t)zig_bit_reverse_u16((uint16_t)(val >> 16), 16) >> 0; #endif return zig_wrap_u32(full_res >> (32 - bits), bits); } static inline int32_t zig_bit_reverse_i32(int32_t val, uint8_t bits) { return zig_wrap_i32((int32_t)zig_bit_reverse_u32((uint32_t)val, bits), bits); } static inline uint64_t zig_bit_reverse_u64(uint64_t val, uint8_t bits) { uint64_t full_res; #if zig_has_builtin(bitreverse64) full_res = __builtin_bitreverse64(val); #else full_res = (uint64_t)zig_bit_reverse_u32((uint32_t)(val >> 0), 32) << 32 | (uint64_t)zig_bit_reverse_u32((uint32_t)(val >> 32), 32) >> 0; #endif return zig_wrap_u64(full_res >> (64 - bits), bits); } static inline int64_t zig_bit_reverse_i64(int64_t val, uint8_t bits) { return zig_wrap_i64((int64_t)zig_bit_reverse_u64((uint64_t)val, bits), bits); } #define zig_builtin_popcount_common(w) \ static inline uint8_t zig_popcount_i##w(int##w##_t val, uint8_t bits) { \ return zig_popcount_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(popcount) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_popcount(w) \ static inline uint8_t zig_popcount_u##w(uint##w##_t val, uint8_t bits) { \ (void)bits; \ return zig_builtin##w(popcount, val); \ } \ \ zig_builtin_popcount_common(w) #else #define zig_builtin_popcount(w) \ static inline uint8_t zig_popcount_u##w(uint##w##_t val, uint8_t bits) { \ (void)bits; \ uint##w##_t temp = val - ((val >> 1) & (UINT##w##_MAX / 3)); \ temp = (temp & (UINT##w##_MAX / 5)) + ((temp >> 2) & (UINT##w##_MAX / 5)); \ temp = (temp + (temp >> 4)) & (UINT##w##_MAX / 17); \ return temp * (UINT##w##_MAX / 255) >> (UINT8_C(w) - UINT8_C(8)); \ } \ \ zig_builtin_popcount_common(w) #endif zig_builtin_popcount(8) zig_builtin_popcount(16) zig_builtin_popcount(32) zig_builtin_popcount(64) #define zig_builtin_ctz_common(w) \ static inline uint8_t zig_ctz_i##w(int##w##_t val, uint8_t bits) { \ return zig_ctz_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(ctz) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_ctz(w) \ static inline uint8_t zig_ctz_u##w(uint##w##_t val, uint8_t bits) { \ if (val == 0) return bits; \ return zig_builtin##w(ctz, val); \ } \ \ zig_builtin_ctz_common(w) #else #define zig_builtin_ctz(w) \ static inline uint8_t zig_ctz_u##w(uint##w##_t val, uint8_t bits) { \ return zig_popcount_u##w(zig_not_u##w(val, bits) & zig_subw_u##w(val, 1, bits), bits); \ } \ \ zig_builtin_ctz_common(w) #endif zig_builtin_ctz(8) zig_builtin_ctz(16) zig_builtin_ctz(32) zig_builtin_ctz(64) #define zig_builtin_clz_common(w) \ static inline uint8_t zig_clz_i##w(int##w##_t val, uint8_t bits) { \ return zig_clz_u##w((uint##w##_t)val, bits); \ } #if zig_has_builtin(clz) || defined(zig_gcc) || defined(zig_tinyc) #define zig_builtin_clz(w) \ static inline uint8_t zig_clz_u##w(uint##w##_t val, uint8_t bits) { \ if (val == 0) return bits; \ return zig_builtin##w(clz, val) - (zig_bitSizeOf(zig_Builtin##w) - bits); \ } \ \ zig_builtin_clz_common(w) #else #define zig_builtin_clz(w) \ static inline uint8_t zig_clz_u##w(uint##w##_t val, uint8_t bits) { \ return zig_ctz_u##w(zig_bit_reverse_u##w(val, bits), bits); \ } \ \ zig_builtin_clz_common(w) #endif zig_builtin_clz(8) zig_builtin_clz(16) zig_builtin_clz(32) zig_builtin_clz(64) /* ======================== 128-bit Integer Support ========================= */ #if !defined(zig_has_int128) # if defined(__SIZEOF_INT128__) # define zig_has_int128 1 # else # define zig_has_int128 0 # endif #endif #if zig_has_int128 typedef unsigned __int128 zig_u128; typedef signed __int128 zig_i128; #define zig_make_u128(hi, lo) ((zig_u128)(hi)<<64|(lo)) #define zig_make_i128(hi, lo) ((zig_i128)zig_make_u128(hi, lo)) #define zig_init_u128(hi, lo) zig_make_u128(hi, lo) #define zig_init_i128(hi, lo) zig_make_i128(hi, lo) #define zig_hi_u128(val) ((uint64_t)((val) >> 64)) #define zig_lo_u128(val) ((uint64_t)((val) >> 0)) #define zig_hi_i128(val) (( int64_t)((val) >> 64)) #define zig_lo_i128(val) ((uint64_t)((val) >> 0)) #define zig_bitCast_u128(val) ((zig_u128)(val)) #define zig_bitCast_i128(val) ((zig_i128)(val)) #define zig_cmp_int128(Type) \ static inline int32_t zig_cmp_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (lhs > rhs) - (lhs < rhs); \ } #define zig_bit_int128(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return lhs operator rhs; \ } #else /* zig_has_int128 */ #if zig_little_endian typedef struct { zig_align(16) uint64_t lo; uint64_t hi; } zig_u128; typedef struct { zig_align(16) uint64_t lo; int64_t hi; } zig_i128; #else typedef struct { zig_align(16) uint64_t hi; uint64_t lo; } zig_u128; typedef struct { zig_align(16) int64_t hi; uint64_t lo; } zig_i128; #endif #define zig_make_u128(hi, lo) ((zig_u128){ .h##i = (hi), .l##o = (lo) }) #define zig_make_i128(hi, lo) ((zig_i128){ .h##i = (hi), .l##o = (lo) }) #if defined(zig_msvc) /* MSVC doesn't allow struct literals in constant expressions */ #define zig_init_u128(hi, lo) { .h##i = (hi), .l##o = (lo) } #define zig_init_i128(hi, lo) { .h##i = (hi), .l##o = (lo) } #else /* But non-MSVC doesn't like the unprotected commas */ #define zig_init_u128(hi, lo) zig_make_u128(hi, lo) #define zig_init_i128(hi, lo) zig_make_i128(hi, lo) #endif #define zig_hi_u128(val) ((val).hi) #define zig_lo_u128(val) ((val).lo) #define zig_hi_i128(val) ((val).hi) #define zig_lo_i128(val) ((val).lo) #define zig_bitCast_u128(val) zig_make_u128((uint64_t)(val).hi, (val).lo) #define zig_bitCast_i128(val) zig_make_i128(( int64_t)(val).hi, (val).lo) #define zig_cmp_int128(Type) \ static inline int32_t zig_cmp_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (lhs.hi == rhs.hi) \ ? (lhs.lo > rhs.lo) - (lhs.lo < rhs.lo) \ : (lhs.hi > rhs.hi) - (lhs.hi < rhs.hi); \ } #define zig_bit_int128(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (zig_##Type){ .hi = lhs.hi operator rhs.hi, .lo = lhs.lo operator rhs.lo }; \ } #endif /* zig_has_int128 */ #define zig_minInt_u128 zig_make_u128(zig_minInt_u64, zig_minInt_u64) #define zig_maxInt_u128 zig_make_u128(zig_maxInt_u64, zig_maxInt_u64) #define zig_minInt_i128 zig_make_i128(zig_minInt_i64, zig_minInt_u64) #define zig_maxInt_i128 zig_make_i128(zig_maxInt_i64, zig_maxInt_u64) zig_cmp_int128(u128) zig_cmp_int128(i128) zig_bit_int128(u128, and, &) zig_bit_int128(i128, and, &) zig_bit_int128(u128, or, |) zig_bit_int128(i128, or, |) zig_bit_int128(u128, xor, ^) zig_bit_int128(i128, xor, ^) static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs); #if zig_has_int128 static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) { return val ^ zig_maxInt_u(128, bits); } static inline zig_i128 zig_not_i128(zig_i128 val, uint8_t bits) { (void)bits; return ~val; } static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs) { return lhs >> rhs; } static inline zig_u128 zig_shl_u128(zig_u128 lhs, uint8_t rhs) { return lhs << rhs; } static inline zig_i128 zig_shr_i128(zig_i128 lhs, uint8_t rhs) { zig_i128 sign_mask = lhs < zig_make_i128(0, 0) ? -zig_make_i128(0, 1) : zig_make_i128(0, 0); return ((lhs ^ sign_mask) >> rhs) ^ sign_mask; } static inline zig_i128 zig_shl_i128(zig_i128 lhs, uint8_t rhs) { return lhs << rhs; } static inline zig_u128 zig_add_u128(zig_u128 lhs, zig_u128 rhs) { return lhs + rhs; } static inline zig_i128 zig_add_i128(zig_i128 lhs, zig_i128 rhs) { return lhs + rhs; } static inline zig_u128 zig_sub_u128(zig_u128 lhs, zig_u128 rhs) { return lhs - rhs; } static inline zig_i128 zig_sub_i128(zig_i128 lhs, zig_i128 rhs) { return lhs - rhs; } static inline zig_u128 zig_mul_u128(zig_u128 lhs, zig_u128 rhs) { return lhs * rhs; } static inline zig_i128 zig_mul_i128(zig_i128 lhs, zig_i128 rhs) { return lhs * rhs; } static inline zig_u128 zig_div_trunc_u128(zig_u128 lhs, zig_u128 rhs) { return lhs / rhs; } static inline zig_i128 zig_div_trunc_i128(zig_i128 lhs, zig_i128 rhs) { return lhs / rhs; } static inline zig_u128 zig_rem_u128(zig_u128 lhs, zig_u128 rhs) { return lhs % rhs; } static inline zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { return lhs % rhs; } #else /* zig_has_int128 */ static inline zig_u128 zig_not_u128(zig_u128 val, uint8_t bits) { return (zig_u128){ .hi = zig_not_u64(val.hi, bits - UINT8_C(64)), .lo = zig_not_u64(val.lo, UINT8_C(64)) }; } static inline zig_i128 zig_not_i128(zig_i128 val, uint8_t bits) { return (zig_i128){ .hi = zig_not_i64(val.hi, bits - UINT8_C(64)), .lo = zig_not_u64(val.lo, UINT8_C(64)) }; } static inline zig_u128 zig_shr_u128(zig_u128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_u128){ .hi = zig_minInt_u64, .lo = lhs.hi >> (rhs - UINT8_C(64)) }; return (zig_u128){ .hi = lhs.hi >> rhs, .lo = lhs.hi << (UINT8_C(64) - rhs) | lhs.lo >> rhs }; } static inline zig_u128 zig_shl_u128(zig_u128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_u128){ .hi = lhs.lo << (rhs - UINT8_C(64)), .lo = zig_minInt_u64 }; return (zig_u128){ .hi = lhs.hi << rhs | lhs.lo >> (UINT8_C(64) - rhs), .lo = lhs.lo << rhs }; } static inline zig_i128 zig_shr_i128(zig_i128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_i128){ .hi = zig_shr_i64(lhs.hi, 63), .lo = zig_shr_i64(lhs.hi, (rhs - UINT8_C(64))) }; return (zig_i128){ .hi = zig_shr_i64(lhs.hi, rhs), .lo = lhs.lo >> rhs | (uint64_t)lhs.hi << (UINT8_C(64) - rhs) }; } static inline zig_i128 zig_shl_i128(zig_i128 lhs, uint8_t rhs) { if (rhs == UINT8_C(0)) return lhs; if (rhs >= UINT8_C(64)) return (zig_i128){ .hi = lhs.lo << (rhs - UINT8_C(64)), .lo = zig_minInt_u64 }; return (zig_i128){ .hi = lhs.hi << rhs | lhs.lo >> (UINT8_C(64) - rhs), .lo = lhs.lo << rhs }; } static inline zig_u128 zig_add_u128(zig_u128 lhs, zig_u128 rhs) { zig_u128 res; res.hi = lhs.hi + rhs.hi + zig_addo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_i128 zig_add_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 res; res.hi = lhs.hi + rhs.hi + zig_addo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_u128 zig_sub_u128(zig_u128 lhs, zig_u128 rhs) { zig_u128 res; res.hi = lhs.hi - rhs.hi - zig_subo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } static inline zig_i128 zig_sub_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 res; res.hi = lhs.hi - rhs.hi - zig_subo_u64(&res.lo, lhs.lo, rhs.lo, 64); return res; } zig_extern zig_i128 __multi3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_mul_i128(zig_i128 lhs, zig_i128 rhs) { return __multi3(lhs, rhs); } static zig_u128 zig_mul_u128(zig_u128 lhs, zig_u128 rhs) { return zig_bitCast_u128(zig_mul_i128(zig_bitCast_i128(lhs), zig_bitCast_i128(rhs))); } zig_extern zig_u128 __udivti3(zig_u128 lhs, zig_u128 rhs); static zig_u128 zig_div_trunc_u128(zig_u128 lhs, zig_u128 rhs) { return __udivti3(lhs, rhs); } zig_extern zig_i128 __divti3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_div_trunc_i128(zig_i128 lhs, zig_i128 rhs) { return __divti3(lhs, rhs); } zig_extern zig_u128 __umodti3(zig_u128 lhs, zig_u128 rhs); static zig_u128 zig_rem_u128(zig_u128 lhs, zig_u128 rhs) { return __umodti3(lhs, rhs); } zig_extern zig_i128 __modti3(zig_i128 lhs, zig_i128 rhs); static zig_i128 zig_rem_i128(zig_i128 lhs, zig_i128 rhs) { return __modti3(lhs, rhs); } #endif /* zig_has_int128 */ #define zig_div_floor_u128 zig_div_trunc_u128 static inline zig_i128 zig_div_floor_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 rem = zig_rem_i128(lhs, rhs); int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); return zig_add_i128(zig_div_trunc_i128(lhs, rhs), zig_make_i128(mask, (uint64_t)mask)); } #define zig_mod_u128 zig_rem_u128 static inline zig_i128 zig_mod_i128(zig_i128 lhs, zig_i128 rhs) { zig_i128 rem = zig_rem_i128(lhs, rhs); int64_t mask = zig_or_u64((uint64_t)zig_hi_i128(rem), zig_lo_i128(rem)) != UINT64_C(0) ? zig_shr_i64(zig_xor_i64(zig_hi_i128(lhs), zig_hi_i128(rhs)), UINT8_C(63)) : INT64_C(0); return zig_add_i128(rem, zig_and_i128(rhs, zig_make_i128(mask, (uint64_t)mask))); } static inline zig_u128 zig_min_u128(zig_u128 lhs, zig_u128 rhs) { return zig_cmp_u128(lhs, rhs) < INT32_C(0) ? lhs : rhs; } static inline zig_i128 zig_min_i128(zig_i128 lhs, zig_i128 rhs) { return zig_cmp_i128(lhs, rhs) < INT32_C(0) ? lhs : rhs; } static inline zig_u128 zig_max_u128(zig_u128 lhs, zig_u128 rhs) { return zig_cmp_u128(lhs, rhs) > INT32_C(0) ? lhs : rhs; } static inline zig_i128 zig_max_i128(zig_i128 lhs, zig_i128 rhs) { return zig_cmp_i128(lhs, rhs) > INT32_C(0) ? lhs : rhs; } static inline zig_u128 zig_wrap_u128(zig_u128 val, uint8_t bits) { return zig_and_u128(val, zig_maxInt_u(128, bits)); } static inline zig_i128 zig_wrap_i128(zig_i128 val, uint8_t bits) { if (bits > UINT8_C(64)) return zig_make_i128(zig_wrap_i64(zig_hi_i128(val), bits - UINT8_C(64)), zig_lo_i128(val)); int64_t lo = zig_wrap_i64((int64_t)zig_lo_i128(val), bits); return zig_make_i128(zig_shr_i64(lo, 63), (uint64_t)lo); } static inline zig_u128 zig_shlw_u128(zig_u128 lhs, uint8_t rhs, uint8_t bits) { return zig_wrap_u128(zig_shl_u128(lhs, rhs), bits); } static inline zig_i128 zig_shlw_i128(zig_i128 lhs, uint8_t rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_shl_u128(zig_bitCast_u128(lhs), rhs)), bits); } static inline zig_u128 zig_addw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_add_u128(lhs, rhs), bits); } static inline zig_i128 zig_addw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_add_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_subw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_sub_u128(lhs, rhs), bits); } static inline zig_i128 zig_subw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_sub_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_mulw_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { return zig_wrap_u128(zig_mul_u128(lhs, rhs), bits); } static inline zig_i128 zig_mulw_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { return zig_wrap_i128(zig_bitCast_i128(zig_mul_u128(zig_bitCast_u128(lhs), zig_bitCast_u128(rhs))), bits); } static inline zig_u128 zig_abs_i128(zig_i128 val) { zig_i128 tmp = zig_shr_i128(val, 127); return zig_bitCast_u128(zig_sub_i128(zig_xor_i128(val, tmp), tmp)); } #if zig_has_int128 static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) zig_u128 full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_addw_u128(lhs, rhs, bits); return *res < lhs; #endif } zig_extern zig_i128 __addoti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(add_overflow) zig_i128 full_res; bool overflow = __builtin_add_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __addoti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) zig_u128 full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_subw_u128(lhs, rhs, bits); return *res > lhs; #endif } zig_extern zig_i128 __suboti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(sub_overflow) zig_i128 full_res; bool overflow = __builtin_sub_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __suboti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } static inline bool zig_mulo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) zig_u128 full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); *res = zig_wrap_u128(full_res, bits); return overflow || full_res < zig_minInt_u(128, bits) || full_res > zig_maxInt_u(128, bits); #else *res = zig_mulw_u128(lhs, rhs, bits); return rhs != zig_make_u128(0, 0) && lhs > zig_maxInt_u(128, bits) / rhs; #endif } zig_extern zig_i128 __muloti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_mulo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { #if zig_has_builtin(mul_overflow) zig_i128 full_res; bool overflow = __builtin_mul_overflow(lhs, rhs, &full_res); #else int overflow_int; zig_i128 full_res = __muloti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0; #endif *res = zig_wrap_i128(full_res, bits); return overflow || full_res < zig_minInt_i(128, bits) || full_res > zig_maxInt_i(128, bits); } #else /* zig_has_int128 */ static inline bool zig_addo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { uint64_t hi; bool overflow = zig_addo_u64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_addo_u64(&res->hi, hi, zig_addo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_addo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int64_t hi; bool overflow = zig_addo_i64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_addo_i64(&res->hi, hi, zig_addo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_subo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { uint64_t hi; bool overflow = zig_subo_u64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_subo_u64(&res->hi, hi, zig_subo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_subo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int64_t hi; bool overflow = zig_subo_i64(&hi, lhs.hi, rhs.hi, bits - 64); return overflow ^ zig_subo_i64(&res->hi, hi, zig_subo_u64(&res->lo, lhs.lo, rhs.lo, 64), bits - 64); } static inline bool zig_mulo_u128(zig_u128 *res, zig_u128 lhs, zig_u128 rhs, uint8_t bits) { *res = zig_mulw_u128(lhs, rhs, bits); return zig_cmp_u128(*res, zig_make_u128(0, 0)) != INT32_C(0) && zig_cmp_u128(lhs, zig_div_trunc_u128(zig_maxInt_u(128, bits), rhs)) > INT32_C(0); } zig_extern zig_i128 __muloti4(zig_i128 lhs, zig_i128 rhs, int *overflow); static inline bool zig_mulo_i128(zig_i128 *res, zig_i128 lhs, zig_i128 rhs, uint8_t bits) { int overflow_int; zig_i128 full_res = __muloti4(lhs, rhs, &overflow_int); bool overflow = overflow_int != 0 || zig_cmp_i128(full_res, zig_minInt_i(128, bits)) < INT32_C(0) || zig_cmp_i128(full_res, zig_maxInt_i(128, bits)) > INT32_C(0); *res = zig_wrap_i128(full_res, bits); return overflow; } #endif /* zig_has_int128 */ static inline bool zig_shlo_u128(zig_u128 *res, zig_u128 lhs, uint8_t rhs, uint8_t bits) { *res = zig_shlw_u128(lhs, rhs, bits); return zig_cmp_u128(lhs, zig_shr_u128(zig_maxInt_u(128, bits), rhs)) > INT32_C(0); } static inline bool zig_shlo_i128(zig_i128 *res, zig_i128 lhs, uint8_t rhs, uint8_t bits) { *res = zig_shlw_i128(lhs, rhs, bits); zig_i128 mask = zig_bitCast_i128(zig_shl_u128(zig_maxInt_u128, bits - rhs - UINT8_C(1))); return zig_cmp_i128(zig_and_i128(lhs, mask), zig_make_i128(0, 0)) != INT32_C(0) && zig_cmp_i128(zig_and_i128(lhs, mask), mask) != INT32_C(0); } static inline zig_u128 zig_shls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; if (zig_cmp_u128(rhs, zig_make_u128(0, bits)) >= INT32_C(0)) return zig_cmp_u128(lhs, zig_make_u128(0, 0)) != INT32_C(0) ? zig_maxInt_u(128, bits) : lhs; return zig_shlo_u128(&res, lhs, (uint8_t)zig_lo_u128(rhs), bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_shls_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (zig_cmp_u128(zig_bitCast_u128(rhs), zig_make_u128(0, bits)) < INT32_C(0) && !zig_shlo_i128(&res, lhs, (uint8_t)zig_lo_i128(rhs), bits)) return res; return zig_cmp_i128(lhs, zig_make_i128(0, 0)) < INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_adds_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_addo_u128(&res, lhs, rhs, bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_adds_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_addo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(res, zig_make_i128(0, 0)) >= INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_subs_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_subo_u128(&res, lhs, rhs, bits) ? zig_minInt_u(128, bits) : res; } static inline zig_i128 zig_subs_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_subo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(res, zig_make_i128(0, 0)) >= INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline zig_u128 zig_muls_u128(zig_u128 lhs, zig_u128 rhs, uint8_t bits) { zig_u128 res; return zig_mulo_u128(&res, lhs, rhs, bits) ? zig_maxInt_u(128, bits) : res; } static inline zig_i128 zig_muls_i128(zig_i128 lhs, zig_i128 rhs, uint8_t bits) { zig_i128 res; if (!zig_mulo_i128(&res, lhs, rhs, bits)) return res; return zig_cmp_i128(zig_xor_i128(lhs, rhs), zig_make_i128(0, 0)) < INT32_C(0) ? zig_minInt_i(128, bits) : zig_maxInt_i(128, bits); } static inline uint8_t zig_clz_u128(zig_u128 val, uint8_t bits) { if (bits <= UINT8_C(64)) return zig_clz_u64(zig_lo_u128(val), bits); if (zig_hi_u128(val) != 0) return zig_clz_u64(zig_hi_u128(val), bits - UINT8_C(64)); return zig_clz_u64(zig_lo_u128(val), UINT8_C(64)) + (bits - UINT8_C(64)); } static inline uint8_t zig_clz_i128(zig_i128 val, uint8_t bits) { return zig_clz_u128(zig_bitCast_u128(val), bits); } static inline uint8_t zig_ctz_u128(zig_u128 val, uint8_t bits) { if (zig_lo_u128(val) != 0) return zig_ctz_u64(zig_lo_u128(val), UINT8_C(64)); return zig_ctz_u64(zig_hi_u128(val), bits - UINT8_C(64)) + UINT8_C(64); } static inline uint8_t zig_ctz_i128(zig_i128 val, uint8_t bits) { return zig_ctz_u128(zig_bitCast_u128(val), bits); } static inline uint8_t zig_popcount_u128(zig_u128 val, uint8_t bits) { return zig_popcount_u64(zig_hi_u128(val), bits - UINT8_C(64)) + zig_popcount_u64(zig_lo_u128(val), UINT8_C(64)); } static inline uint8_t zig_popcount_i128(zig_i128 val, uint8_t bits) { return zig_popcount_u128(zig_bitCast_u128(val), bits); } static inline zig_u128 zig_byte_swap_u128(zig_u128 val, uint8_t bits) { zig_u128 full_res; #if zig_has_builtin(bswap128) full_res = __builtin_bswap128(val); #else full_res = zig_make_u128(zig_byte_swap_u64(zig_lo_u128(val), UINT8_C(64)), zig_byte_swap_u64(zig_hi_u128(val), UINT8_C(64))); #endif return zig_shr_u128(full_res, UINT8_C(128) - bits); } static inline zig_i128 zig_byte_swap_i128(zig_i128 val, uint8_t bits) { return zig_bitCast_i128(zig_byte_swap_u128(zig_bitCast_u128(val), bits)); } static inline zig_u128 zig_bit_reverse_u128(zig_u128 val, uint8_t bits) { return zig_shr_u128(zig_make_u128(zig_bit_reverse_u64(zig_lo_u128(val), UINT8_C(64)), zig_bit_reverse_u64(zig_hi_u128(val), UINT8_C(64))), UINT8_C(128) - bits); } static inline zig_i128 zig_bit_reverse_i128(zig_i128 val, uint8_t bits) { return zig_bitCast_i128(zig_bit_reverse_u128(zig_bitCast_u128(val), bits)); } /* ========================== Big Integer Support =========================== */ static inline uint16_t zig_int_bytes(uint16_t bits) { uint16_t bytes = (bits + CHAR_BIT - 1) / CHAR_BIT; uint16_t alignment = ZIG_TARGET_MAX_INT_ALIGNMENT; while (alignment / 2 >= bytes) alignment /= 2; return (bytes + alignment - 1) / alignment * alignment; } static inline int32_t zig_cmp_big(const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; bool do_signed = is_signed; uint16_t remaining_bytes = zig_int_bytes(bits); #if zig_little_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { int32_t limb_cmp; #if zig_little_endian byte_offset -= 128 / CHAR_BIT; #endif if (do_signed) { zig_i128 lhs_limb; zig_i128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_cmp = zig_cmp_i128(lhs_limb, rhs_limb); do_signed = false; } else { zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_cmp = zig_cmp_u128(lhs_limb, rhs_limb); } if (limb_cmp != 0) return limb_cmp; remaining_bytes -= 128 / CHAR_BIT; #if zig_big_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_little_endian byte_offset -= 64 / CHAR_BIT; #endif if (do_signed) { int64_t lhs_limb; int64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 64 / CHAR_BIT; #if zig_big_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_little_endian byte_offset -= 32 / CHAR_BIT; #endif if (do_signed) { int32_t lhs_limb; int32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 32 / CHAR_BIT; #if zig_big_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_little_endian byte_offset -= 16 / CHAR_BIT; #endif if (do_signed) { int16_t lhs_limb; int16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 16 / CHAR_BIT; #if zig_big_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_little_endian byte_offset -= 8 / CHAR_BIT; #endif if (do_signed) { int8_t lhs_limb; int8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); do_signed = false; } else { uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); if (lhs_limb != rhs_limb) return (lhs_limb > rhs_limb) - (lhs_limb < rhs_limb); } remaining_bytes -= 8 / CHAR_BIT; #if zig_big_endian byte_offset += 8 / CHAR_BIT; #endif } return 0; } static inline void zig_and_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_and_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline void zig_or_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_or_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline void zig_xor_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); (void)is_signed; while (remaining_bytes >= 128 / CHAR_BIT) { zig_u128 res_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u128(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 128 / CHAR_BIT; byte_offset += 128 / CHAR_BIT; } while (remaining_bytes >= 64 / CHAR_BIT) { uint64_t res_limb; uint64_t lhs_limb; uint64_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u64(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 64 / CHAR_BIT; byte_offset += 64 / CHAR_BIT; } while (remaining_bytes >= 32 / CHAR_BIT) { uint32_t res_limb; uint32_t lhs_limb; uint32_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u32(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 32 / CHAR_BIT; byte_offset += 32 / CHAR_BIT; } while (remaining_bytes >= 16 / CHAR_BIT) { uint16_t res_limb; uint16_t lhs_limb; uint16_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u16(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 16 / CHAR_BIT; byte_offset += 16 / CHAR_BIT; } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t res_limb; uint8_t lhs_limb; uint8_t rhs_limb; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); res_limb = zig_xor_u8(lhs_limb, rhs_limb); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); remaining_bytes -= 8 / CHAR_BIT; byte_offset += 8 / CHAR_BIT; } } static inline bool zig_addo_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint8_t top_bits = (uint8_t)(remaining_bytes * 8 - bits); bool overflow = false; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { uint8_t limb_bits = 128 - (remaining_bytes == 128 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif if (remaining_bytes == 128 / CHAR_BIT && is_signed) { zig_i128 res_limb; zig_i128 tmp_limb; zig_i128 lhs_limb; zig_i128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i128(&res_limb, tmp_limb, zig_make_i128(INT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { zig_u128 res_limb; zig_u128 tmp_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u128(&res_limb, tmp_limb, zig_make_u128(UINT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { uint8_t limb_bits = 64 - (remaining_bytes == 64 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif if (remaining_bytes == 64 / CHAR_BIT && is_signed) { int64_t res_limb; int64_t tmp_limb; int64_t lhs_limb; int64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i64(&res_limb, tmp_limb, overflow ? INT64_C(1) : INT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint64_t res_limb; uint64_t tmp_limb; uint64_t lhs_limb; uint64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u64(&res_limb, tmp_limb, overflow ? UINT64_C(1) : UINT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { uint8_t limb_bits = 32 - (remaining_bytes == 32 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif if (remaining_bytes == 32 / CHAR_BIT && is_signed) { int32_t res_limb; int32_t tmp_limb; int32_t lhs_limb; int32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i32(&res_limb, tmp_limb, overflow ? INT32_C(1) : INT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint32_t res_limb; uint32_t tmp_limb; uint32_t lhs_limb; uint32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u32(&res_limb, tmp_limb, overflow ? UINT32_C(1) : UINT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { uint8_t limb_bits = 16 - (remaining_bytes == 16 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif if (remaining_bytes == 16 / CHAR_BIT && is_signed) { int16_t res_limb; int16_t tmp_limb; int16_t lhs_limb; int16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i16(&res_limb, tmp_limb, overflow ? INT16_C(1) : INT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint16_t res_limb; uint16_t tmp_limb; uint16_t lhs_limb; uint16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u16(&res_limb, tmp_limb, overflow ? UINT16_C(1) : UINT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t limb_bits = 8 - (remaining_bytes == 8 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif if (remaining_bytes == 8 / CHAR_BIT && is_signed) { int8_t res_limb; int8_t tmp_limb; int8_t lhs_limb; int8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_i8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_i8(&res_limb, tmp_limb, overflow ? INT8_C(1) : INT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint8_t res_limb; uint8_t tmp_limb; uint8_t lhs_limb; uint8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_addo_u8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_addo_u8(&res_limb, tmp_limb, overflow ? UINT8_C(1) : UINT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return overflow; } static inline bool zig_subo_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { uint8_t *res_bytes = res; const uint8_t *lhs_bytes = lhs; const uint8_t *rhs_bytes = rhs; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint8_t top_bits = (uint8_t)(remaining_bytes * 8 - bits); bool overflow = false; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { uint8_t limb_bits = 128 - (remaining_bytes == 128 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif if (remaining_bytes == 128 / CHAR_BIT && is_signed) { zig_i128 res_limb; zig_i128 tmp_limb; zig_i128 lhs_limb; zig_i128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i128(&res_limb, tmp_limb, zig_make_i128(INT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { zig_u128 res_limb; zig_u128 tmp_limb; zig_u128 lhs_limb; zig_u128 rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u128(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u128(&res_limb, tmp_limb, zig_make_u128(UINT64_C(0), overflow ? UINT64_C(1) : UINT64_C(0)), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { uint8_t limb_bits = 64 - (remaining_bytes == 64 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif if (remaining_bytes == 64 / CHAR_BIT && is_signed) { int64_t res_limb; int64_t tmp_limb; int64_t lhs_limb; int64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i64(&res_limb, tmp_limb, overflow ? INT64_C(1) : INT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint64_t res_limb; uint64_t tmp_limb; uint64_t lhs_limb; uint64_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u64(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u64(&res_limb, tmp_limb, overflow ? UINT64_C(1) : UINT64_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { uint8_t limb_bits = 32 - (remaining_bytes == 32 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif if (remaining_bytes == 32 / CHAR_BIT && is_signed) { int32_t res_limb; int32_t tmp_limb; int32_t lhs_limb; int32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i32(&res_limb, tmp_limb, overflow ? INT32_C(1) : INT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint32_t res_limb; uint32_t tmp_limb; uint32_t lhs_limb; uint32_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u32(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u32(&res_limb, tmp_limb, overflow ? UINT32_C(1) : UINT32_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { uint8_t limb_bits = 16 - (remaining_bytes == 16 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif if (remaining_bytes == 16 / CHAR_BIT && is_signed) { int16_t res_limb; int16_t tmp_limb; int16_t lhs_limb; int16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i16(&res_limb, tmp_limb, overflow ? INT16_C(1) : INT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint16_t res_limb; uint16_t tmp_limb; uint16_t lhs_limb; uint16_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u16(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u16(&res_limb, tmp_limb, overflow ? UINT16_C(1) : UINT16_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { uint8_t limb_bits = 8 - (remaining_bytes == 8 / CHAR_BIT ? top_bits : 0); #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif if (remaining_bytes == 8 / CHAR_BIT && is_signed) { int8_t res_limb; int8_t tmp_limb; int8_t lhs_limb; int8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_i8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_i8(&res_limb, tmp_limb, overflow ? INT8_C(1) : INT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } else { uint8_t res_limb; uint8_t tmp_limb; uint8_t lhs_limb; uint8_t rhs_limb; bool limb_overflow; memcpy(&lhs_limb, &lhs_bytes[byte_offset], sizeof(lhs_limb)); memcpy(&rhs_limb, &rhs_bytes[byte_offset], sizeof(rhs_limb)); limb_overflow = zig_subo_u8(&tmp_limb, lhs_limb, rhs_limb, limb_bits); overflow = limb_overflow ^ zig_subo_u8(&res_limb, tmp_limb, overflow ? UINT8_C(1) : UINT8_C(0), limb_bits); memcpy(&res_bytes[byte_offset], &res_limb, sizeof(res_limb)); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return overflow; } static inline void zig_addw_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { (void)zig_addo_big(res, lhs, rhs, is_signed, bits); } static inline void zig_subw_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { (void)zig_subo_big(res, lhs, rhs, is_signed, bits); } zig_extern void __udivei4(uint32_t *res, const uint32_t *lhs, const uint32_t *rhs, uintptr_t bits); static inline void zig_div_trunc_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { __udivei4(res, lhs, rhs, bits); return; } zig_trap(); } static inline void zig_div_floor_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { zig_div_trunc_big(res, lhs, rhs, is_signed, bits); return; } zig_trap(); } zig_extern void __umodei4(uint32_t *res, const uint32_t *lhs, const uint32_t *rhs, uintptr_t bits); static inline void zig_rem_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { __umodei4(res, lhs, rhs, bits); return; } zig_trap(); } static inline void zig_mod_big(void *res, const void *lhs, const void *rhs, bool is_signed, uint16_t bits) { if (!is_signed) { zig_rem_big(res, lhs, rhs, is_signed, bits); return; } zig_trap(); } static inline uint16_t zig_clz_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t skip_bits = remaining_bytes * 8 - bits; uint16_t total_lz = 0; uint16_t limb_lz; (void)is_signed; #if zig_little_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_little_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u128(val_limb, 128 - skip_bits); } total_lz += limb_lz; if (limb_lz < 128 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 128 / CHAR_BIT; #if zig_big_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_little_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u64(val_limb, 64 - skip_bits); } total_lz += limb_lz; if (limb_lz < 64 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 64 / CHAR_BIT; #if zig_big_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_little_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u32(val_limb, 32 - skip_bits); } total_lz += limb_lz; if (limb_lz < 32 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 32 / CHAR_BIT; #if zig_big_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_little_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u16(val_limb, 16 - skip_bits); } total_lz += limb_lz; if (limb_lz < 16 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 16 / CHAR_BIT; #if zig_big_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_little_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_lz = zig_clz_u8(val_limb, 8 - skip_bits); } total_lz += limb_lz; if (limb_lz < 8 - skip_bits) return total_lz; skip_bits = 0; remaining_bytes -= 8 / CHAR_BIT; #if zig_big_endian byte_offset += 8 / CHAR_BIT; #endif } return total_lz; } static inline uint16_t zig_ctz_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t total_tz = 0; uint16_t limb_tz; (void)is_signed; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u128(val_limb, 128); } total_tz += limb_tz; if (limb_tz < 128) return total_tz; remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u64(val_limb, 64); } total_tz += limb_tz; if (limb_tz < 64) return total_tz; remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u32(val_limb, 32); } total_tz += limb_tz; if (limb_tz < 32) return total_tz; remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u16(val_limb, 16); } total_tz += limb_tz; if (limb_tz < 16) return total_tz; remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); limb_tz = zig_ctz_u8(val_limb, 8); } total_tz += limb_tz; if (limb_tz < 8) return total_tz; remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return total_tz; } static inline uint16_t zig_popcount_big(const void *val, bool is_signed, uint16_t bits) { const uint8_t *val_bytes = val; uint16_t byte_offset = 0; uint16_t remaining_bytes = zig_int_bytes(bits); uint16_t total_pc = 0; (void)is_signed; #if zig_big_endian byte_offset = remaining_bytes; #endif while (remaining_bytes >= 128 / CHAR_BIT) { #if zig_big_endian byte_offset -= 128 / CHAR_BIT; #endif { zig_u128 val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u128(val_limb, 128); } remaining_bytes -= 128 / CHAR_BIT; #if zig_little_endian byte_offset += 128 / CHAR_BIT; #endif } while (remaining_bytes >= 64 / CHAR_BIT) { #if zig_big_endian byte_offset -= 64 / CHAR_BIT; #endif { uint64_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u64(val_limb, 64); } remaining_bytes -= 64 / CHAR_BIT; #if zig_little_endian byte_offset += 64 / CHAR_BIT; #endif } while (remaining_bytes >= 32 / CHAR_BIT) { #if zig_big_endian byte_offset -= 32 / CHAR_BIT; #endif { uint32_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc += zig_popcount_u32(val_limb, 32); } remaining_bytes -= 32 / CHAR_BIT; #if zig_little_endian byte_offset += 32 / CHAR_BIT; #endif } while (remaining_bytes >= 16 / CHAR_BIT) { #if zig_big_endian byte_offset -= 16 / CHAR_BIT; #endif { uint16_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc = zig_popcount_u16(val_limb, 16); } remaining_bytes -= 16 / CHAR_BIT; #if zig_little_endian byte_offset += 16 / CHAR_BIT; #endif } while (remaining_bytes >= 8 / CHAR_BIT) { #if zig_big_endian byte_offset -= 8 / CHAR_BIT; #endif { uint8_t val_limb; memcpy(&val_limb, &val_bytes[byte_offset], sizeof(val_limb)); total_pc = zig_popcount_u8(val_limb, 8); } remaining_bytes -= 8 / CHAR_BIT; #if zig_little_endian byte_offset += 8 / CHAR_BIT; #endif } return total_pc; } /* ========================= Floating Point Support ========================= */ #ifndef __STDC_WANT_IEC_60559_TYPES_EXT__ #define __STDC_WANT_IEC_60559_TYPES_EXT__ #endif #include #if defined(zig_msvc) float __cdecl nanf(char const* input); double __cdecl nan(char const* input); long double __cdecl nanl(char const* input); #define zig_msvc_flt_inf ((double)(1e+300 * 1e+300)) #define zig_msvc_flt_inff ((float)(1e+300 * 1e+300)) #define zig_msvc_flt_infl ((long double)(1e+300 * 1e+300)) #define zig_msvc_flt_nan ((double)(zig_msvc_flt_inf * 0.f)) #define zig_msvc_flt_nanf ((float)(zig_msvc_flt_inf * 0.f)) #define zig_msvc_flt_nanl ((long double)(zig_msvc_flt_inf * 0.f)) #define __builtin_nan(str) nan(str) #define __builtin_nanf(str) nanf(str) #define __builtin_nanl(str) nanl(str) #define __builtin_inf() zig_msvc_flt_inf #define __builtin_inff() zig_msvc_flt_inff #define __builtin_infl() zig_msvc_flt_infl #endif #if (zig_has_builtin(nan) && zig_has_builtin(nans) && zig_has_builtin(inf)) || defined(zig_gcc) #define zig_make_special_f16(sign, name, arg, repr) sign zig_make_f16 (__builtin_##name, )(arg) #define zig_make_special_f32(sign, name, arg, repr) sign zig_make_f32 (__builtin_##name, )(arg) #define zig_make_special_f64(sign, name, arg, repr) sign zig_make_f64 (__builtin_##name, )(arg) #define zig_make_special_f80(sign, name, arg, repr) sign zig_make_f80 (__builtin_##name, )(arg) #define zig_make_special_f128(sign, name, arg, repr) sign zig_make_f128(__builtin_##name, )(arg) #else #define zig_make_special_f16(sign, name, arg, repr) zig_bitCast_f16 (repr) #define zig_make_special_f32(sign, name, arg, repr) zig_bitCast_f32 (repr) #define zig_make_special_f64(sign, name, arg, repr) zig_bitCast_f64 (repr) #define zig_make_special_f80(sign, name, arg, repr) zig_bitCast_f80 (repr) #define zig_make_special_f128(sign, name, arg, repr) zig_bitCast_f128(repr) #endif #define zig_has_f16 1 #define zig_libc_name_f16(name) __##name##h #define zig_init_special_f16(sign, name, arg, repr) zig_make_special_f16(sign, name, arg, repr) #if FLT_MANT_DIG == 11 typedef float zig_f16; #define zig_make_f16(fp, repr) fp##f #elif DBL_MANT_DIG == 11 typedef double zig_f16; #define zig_make_f16(fp, repr) fp #elif LDBL_MANT_DIG == 11 typedef long double zig_f16; #define zig_make_f16(fp, repr) fp##l #elif FLT16_MANT_DIG == 11 && (zig_has_builtin(inff16) || defined(zig_gcc)) typedef _Float16 zig_f16; #define zig_make_f16(fp, repr) fp##f16 #elif defined(__SIZEOF_FP16__) typedef __fp16 zig_f16; #define zig_make_f16(fp, repr) fp##f16 #else #undef zig_has_f16 #define zig_has_f16 0 #define zig_repr_f16 u16 typedef uint16_t zig_f16; #define zig_make_f16(fp, repr) repr #undef zig_make_special_f16 #define zig_make_special_f16(sign, name, arg, repr) repr #undef zig_init_special_f16 #define zig_init_special_f16(sign, name, arg, repr) repr #endif #if defined(zig_darwin) && defined(zig_x86) typedef uint16_t zig_compiler_rt_f16; #else typedef zig_f16 zig_compiler_rt_f16; #endif #define zig_has_f32 1 #define zig_libc_name_f32(name) name##f #if defined(zig_msvc) #define zig_init_special_f32(sign, name, arg, repr) sign zig_make_f32(zig_msvc_flt_##name, ) #else #define zig_init_special_f32(sign, name, arg, repr) zig_make_special_f32(sign, name, arg, repr) #endif #if FLT_MANT_DIG == 24 typedef float zig_f32; #define zig_make_f32(fp, repr) fp##f #elif DBL_MANT_DIG == 24 typedef double zig_f32; #define zig_make_f32(fp, repr) fp #elif LDBL_MANT_DIG == 24 typedef long double zig_f32; #define zig_make_f32(fp, repr) fp##l #elif FLT32_MANT_DIG == 24 typedef _Float32 zig_f32; #define zig_make_f32(fp, repr) fp##f32 #else #undef zig_has_f32 #define zig_has_f32 0 #define zig_repr_f32 u32 typedef uint32_t zig_f32; #define zig_make_f32(fp, repr) repr #undef zig_make_special_f32 #define zig_make_special_f32(sign, name, arg, repr) repr #undef zig_init_special_f32 #define zig_init_special_f32(sign, name, arg, repr) repr #endif #define zig_has_f64 1 #define zig_libc_name_f64(name) name #if defined(zig_msvc) #define zig_init_special_f64(sign, name, arg, repr) sign zig_make_f64(zig_msvc_flt_##name, ) #else #define zig_init_special_f64(sign, name, arg, repr) zig_make_special_f64(sign, name, arg, repr) #endif #if FLT_MANT_DIG == 53 typedef float zig_f64; #define zig_make_f64(fp, repr) fp##f #elif DBL_MANT_DIG == 53 typedef double zig_f64; #define zig_make_f64(fp, repr) fp #elif LDBL_MANT_DIG == 53 typedef long double zig_f64; #define zig_make_f64(fp, repr) fp##l #elif FLT64_MANT_DIG == 53 typedef _Float64 zig_f64; #define zig_make_f64(fp, repr) fp##f64 #elif FLT32X_MANT_DIG == 53 typedef _Float32x zig_f64; #define zig_make_f64(fp, repr) fp##f32x #else #undef zig_has_f64 #define zig_has_f64 0 #define zig_repr_f64 u64 typedef uint64_t zig_f64; #define zig_make_f64(fp, repr) repr #undef zig_make_special_f64 #define zig_make_special_f64(sign, name, arg, repr) repr #undef zig_init_special_f64 #define zig_init_special_f64(sign, name, arg, repr) repr #endif #define zig_has_f80 1 #define zig_libc_name_f80(name) __##name##x #define zig_init_special_f80(sign, name, arg, repr) zig_make_special_f80(sign, name, arg, repr) #if FLT_MANT_DIG == 64 typedef float zig_f80; #define zig_make_f80(fp, repr) fp##f #elif DBL_MANT_DIG == 64 typedef double zig_f80; #define zig_make_f80(fp, repr) fp #elif LDBL_MANT_DIG == 64 typedef long double zig_f80; #define zig_make_f80(fp, repr) fp##l #elif FLT80_MANT_DIG == 64 typedef _Float80 zig_f80; #define zig_make_f80(fp, repr) fp##f80 #elif FLT64X_MANT_DIG == 64 typedef _Float64x zig_f80; #define zig_make_f80(fp, repr) fp##f64x #elif defined(__SIZEOF_FLOAT80__) typedef __float80 zig_f80; #define zig_make_f80(fp, repr) fp##l #else #undef zig_has_f80 #define zig_has_f80 0 #define zig_repr_f80 u128 typedef zig_u128 zig_f80; #define zig_make_f80(fp, repr) repr #undef zig_make_special_f80 #define zig_make_special_f80(sign, name, arg, repr) repr #undef zig_init_special_f80 #define zig_init_special_f80(sign, name, arg, repr) repr #endif #if defined(zig_gcc) && defined(zig_x86) #define zig_f128_has_miscompilations 1 #else #define zig_f128_has_miscompilations 0 #endif #define zig_has_f128 1 #define zig_libc_name_f128(name) name##q #define zig_init_special_f128(sign, name, arg, repr) zig_make_special_f128(sign, name, arg, repr) #if !zig_f128_has_miscompilations && FLT_MANT_DIG == 113 typedef float zig_f128; #define zig_make_f128(fp, repr) fp##f #elif !zig_f128_has_miscompilations && DBL_MANT_DIG == 113 typedef double zig_f128; #define zig_make_f128(fp, repr) fp #elif !zig_f128_has_miscompilations && LDBL_MANT_DIG == 113 typedef long double zig_f128; #define zig_make_f128(fp, repr) fp##l #elif !zig_f128_has_miscompilations && FLT128_MANT_DIG == 113 typedef _Float128 zig_f128; #define zig_make_f128(fp, repr) fp##f128 #elif !zig_f128_has_miscompilations && FLT64X_MANT_DIG == 113 typedef _Float64x zig_f128; #define zig_make_f128(fp, repr) fp##f64x #elif !zig_f128_has_miscompilations && defined(__SIZEOF_FLOAT128__) typedef __float128 zig_f128; #define zig_make_f128(fp, repr) fp##q #undef zig_make_special_f128 #define zig_make_special_f128(sign, name, arg, repr) sign __builtin_##name##f128(arg) #else #undef zig_has_f128 #define zig_has_f128 0 #undef zig_make_special_f128 #undef zig_init_special_f128 #if defined(zig_darwin) || defined(zig_aarch64) typedef __attribute__((__vector_size__(2 * sizeof(uint64_t)))) uint64_t zig_v2u64; zig_basic_operator(zig_v2u64, xor_v2u64, ^) #define zig_repr_f128 v2u64 typedef zig_v2u64 zig_f128; #define zig_make_f128_zig_make_u128(hi, lo) (zig_f128){ lo, hi } #define zig_make_f128_zig_init_u128 zig_make_f128_zig_make_u128 #define zig_make_f128(fp, repr) zig_make_f128_##repr #define zig_make_special_f128(sign, name, arg, repr) zig_make_f128_##repr #define zig_init_special_f128(sign, name, arg, repr) zig_make_f128_##repr #else #define zig_repr_f128 u128 typedef zig_u128 zig_f128; #define zig_make_f128(fp, repr) repr #define zig_make_special_f128(sign, name, arg, repr) repr #define zig_init_special_f128(sign, name, arg, repr) repr #endif #endif #if !defined(zig_msvc) && defined(ZIG_TARGET_ABI_MSVC) /* Emulate msvc abi on a gnu compiler */ typedef zig_f64 zig_c_longdouble; #elif defined(zig_msvc) && !defined(ZIG_TARGET_ABI_MSVC) /* Emulate gnu abi on an msvc compiler */ typedef zig_f128 zig_c_longdouble; #else /* Target and compiler abi match */ typedef long double zig_c_longdouble; #endif #define zig_bitCast_float(Type, ReprType) \ static inline zig_##Type zig_bitCast_##Type(ReprType repr) { \ zig_##Type result; \ memcpy(&result, &repr, sizeof(result)); \ return result; \ } zig_bitCast_float(f16, uint16_t) zig_bitCast_float(f32, uint32_t) zig_bitCast_float(f64, uint64_t) zig_bitCast_float(f80, zig_u128) zig_bitCast_float(f128, zig_u128) #define zig_convert_builtin(ExternResType, ResType, operation, ExternArgType, ArgType, version) \ zig_extern ExternResType zig_expand_concat(zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType), version)(ExternArgType); \ static inline ResType zig_expand_concat(zig_expand_concat(zig_##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType)(ArgType arg) { \ ResType res; \ ExternResType extern_res; \ ExternArgType extern_arg; \ memcpy(&extern_arg, &arg, sizeof(extern_arg)); \ extern_res = zig_expand_concat(zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_##ArgType), zig_compiler_rt_abbrev_##ResType), version)(extern_arg); \ memcpy(&res, &extern_res, sizeof(res)); \ return extern_res; \ } zig_convert_builtin(zig_compiler_rt_f16, zig_f16, trunc, zig_f32, zig_f32, 2) zig_convert_builtin(zig_compiler_rt_f16, zig_f16, trunc, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f16, zig_f16, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f16, zig_f16, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f32, zig_f32, extend, zig_compiler_rt_f16, zig_f16, 2) zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f64, zig_f64, extend, zig_compiler_rt_f16, zig_f16, 2) zig_convert_builtin(zig_f64, zig_f64, trunc, zig_f80, zig_f80, 2) zig_convert_builtin(zig_f64, zig_f64, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f16, zig_f16, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f32, zig_f32, 2) zig_convert_builtin(zig_f80, zig_f80, extend, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f80, zig_f80, trunc, zig_f128, zig_f128, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f16, zig_f16, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f32, zig_f32, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f128, zig_f128, extend, zig_f80, zig_f80, 2) #ifdef __ARM_EABI__ zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_d2f(zig_f64); static inline zig_f32 zig_truncdfsf(zig_f64 arg) { return __aeabi_d2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_f2d(zig_f32); static inline zig_f64 zig_extendsfdf(zig_f32 arg) { return __aeabi_f2d(arg); } #else /* __ARM_EABI__ */ zig_convert_builtin(zig_f32, zig_f32, trunc, zig_f64, zig_f64, 2) zig_convert_builtin(zig_f64, zig_f64, extend, zig_f32, zig_f32, 2) #endif /* __ARM_EABI__ */ #define zig_float_negate_builtin_0(w, c, sb) \ zig_expand_concat(zig_xor_, zig_repr_f##w)(arg, zig_make_f##w(-0x0.0p0, c sb)) #define zig_float_negate_builtin_1(w, c, sb) -arg #define zig_float_negate_builtin(w, c, sb) \ static inline zig_f##w zig_neg_f##w(zig_f##w arg) { \ return zig_expand_concat(zig_float_negate_builtin_, zig_has_f##w)(w, c, sb); \ } zig_float_negate_builtin(16, , UINT16_C(1) << 15 ) zig_float_negate_builtin(32, , UINT32_C(1) << 31 ) zig_float_negate_builtin(64, , UINT64_C(1) << 63 ) zig_float_negate_builtin(80, zig_make_u128, (UINT64_C(1) << 15, UINT64_C(0))) zig_float_negate_builtin(128, zig_make_u128, (UINT64_C(1) << 63, UINT64_C(0))) #define zig_float_less_builtin_0(Type, operation) \ zig_extern int32_t zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_zig_##Type), 2)(zig_##Type, zig_##Type); \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return zig_expand_concat(zig_expand_concat(__##operation, zig_compiler_rt_abbrev_zig_##Type), 2)(lhs, rhs); \ } #define zig_float_less_builtin_1(Type, operation) \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return (!(lhs <= rhs) - (lhs < rhs)); \ } #define zig_float_greater_builtin_0(Type, operation) \ zig_float_less_builtin_0(Type, operation) #define zig_float_greater_builtin_1(Type, operation) \ static inline int32_t zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return ((lhs > rhs) - !(lhs >= rhs)); \ } #define zig_float_binary_builtin_0(Type, operation, operator) \ zig_extern zig_##Type zig_expand_concat(zig_expand_concat(__##operation, \ zig_compiler_rt_abbrev_zig_##Type), 3)(zig_##Type, zig_##Type); \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return zig_expand_concat(zig_expand_concat(__##operation, zig_compiler_rt_abbrev_zig_##Type), 3)(lhs, rhs); \ } #define zig_float_binary_builtin_1(Type, operation, operator) \ static inline zig_##Type zig_##operation##_##Type(zig_##Type lhs, zig_##Type rhs) { \ return lhs operator rhs; \ } #define zig_common_float_builtins(w) \ zig_convert_builtin( int64_t, int64_t, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_i128, zig_i128, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_u128, zig_u128, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, int64_t, int64_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, zig_i128, zig_i128, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, zig_u128, zig_u128, ) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, cmp) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, ne) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, eq) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, lt) \ zig_expand_concat(zig_float_less_builtin_, zig_has_f##w)(f##w, le) \ zig_expand_concat(zig_float_greater_builtin_, zig_has_f##w)(f##w, gt) \ zig_expand_concat(zig_float_greater_builtin_, zig_has_f##w)(f##w, ge) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, add, +) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, sub, -) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, mul, *) \ zig_expand_concat(zig_float_binary_builtin_, zig_has_f##w)(f##w, div, /) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(sqrt)))(zig_f##w, zig_sqrt_f##w, zig_libc_name_f##w(sqrt), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(sin)))(zig_f##w, zig_sin_f##w, zig_libc_name_f##w(sin), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(cos)))(zig_f##w, zig_cos_f##w, zig_libc_name_f##w(cos), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(tan)))(zig_f##w, zig_tan_f##w, zig_libc_name_f##w(tan), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(exp)))(zig_f##w, zig_exp_f##w, zig_libc_name_f##w(exp), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(exp2)))(zig_f##w, zig_exp2_f##w, zig_libc_name_f##w(exp2), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log)))(zig_f##w, zig_log_f##w, zig_libc_name_f##w(log), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log2)))(zig_f##w, zig_log2_f##w, zig_libc_name_f##w(log2), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(log10)))(zig_f##w, zig_log10_f##w, zig_libc_name_f##w(log10), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fabs)))(zig_f##w, zig_abs_f##w, zig_libc_name_f##w(fabs), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(floor)))(zig_f##w, zig_floor_f##w, zig_libc_name_f##w(floor), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(ceil)))(zig_f##w, zig_ceil_f##w, zig_libc_name_f##w(ceil), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(round)))(zig_f##w, zig_round_f##w, zig_libc_name_f##w(round), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(trunc)))(zig_f##w, zig_trunc_f##w, zig_libc_name_f##w(trunc), (zig_f##w x), (x)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmod)))(zig_f##w, zig_fmod_f##w, zig_libc_name_f##w(fmod), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmin)))(zig_f##w, zig_min_f##w, zig_libc_name_f##w(fmin), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fmax)))(zig_f##w, zig_max_f##w, zig_libc_name_f##w(fmax), (zig_f##w x, zig_f##w y), (x, y)) \ zig_expand_concat(zig_expand_import_, zig_expand_has_builtin(zig_libc_name_f##w(fma)))(zig_f##w, zig_fma_f##w, zig_libc_name_f##w(fma), (zig_f##w x, zig_f##w y, zig_f##w z), (x, y, z)) \ \ static inline zig_f##w zig_div_trunc_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_trunc_f##w(zig_div_f##w(lhs, rhs)); \ } \ \ static inline zig_f##w zig_div_floor_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_floor_f##w(zig_div_f##w(lhs, rhs)); \ } \ \ static inline zig_f##w zig_mod_f##w(zig_f##w lhs, zig_f##w rhs) { \ return zig_sub_f##w(lhs, zig_mul_f##w(zig_div_floor_f##w(lhs, rhs), rhs)); \ } zig_common_float_builtins(16) zig_common_float_builtins(32) zig_common_float_builtins(64) zig_common_float_builtins(80) zig_common_float_builtins(128) #define zig_float_builtins(w) \ zig_convert_builtin( int32_t, int32_t, fix, zig_f##w, zig_f##w, ) \ zig_convert_builtin(uint32_t, uint32_t, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(uint64_t, uint64_t, fixuns, zig_f##w, zig_f##w, ) \ zig_convert_builtin(zig_f##w, zig_f##w, float, int32_t, int32_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, uint32_t, uint32_t, ) \ zig_convert_builtin(zig_f##w, zig_f##w, floatun, uint64_t, uint64_t, ) zig_float_builtins(16) zig_float_builtins(80) zig_float_builtins(128) #ifdef __ARM_EABI__ zig_extern zig_callconv(pcs("aapcs")) int32_t __aeabi_f2iz(zig_f32); static inline int32_t zig_fixsfsi(zig_f32 arg) { return __aeabi_f2iz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint32_t __aeabi_f2uiz(zig_f32); static inline uint32_t zig_fixunssfsi(zig_f32 arg) { return __aeabi_f2uiz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint64_t __aeabi_f2ulz(zig_f32); static inline uint64_t zig_fixunssfdi(zig_f32 arg) { return __aeabi_f2ulz(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_i2f(int32_t); static inline zig_f32 zig_floatsisf(int32_t arg) { return __aeabi_i2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_ui2f(uint32_t); static inline zig_f32 zig_floatunsisf(uint32_t arg) { return __aeabi_ui2f(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f32 __aeabi_ul2f(uint64_t); static inline zig_f32 zig_floatundisf(uint64_t arg) { return __aeabi_ul2f(arg); } zig_extern zig_callconv(pcs("aapcs")) int32_t __aeabi_d2iz(zig_f64); static inline int32_t zig_fixdfsi(zig_f64 arg) { return __aeabi_d2iz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint32_t __aeabi_d2uiz(zig_f64); static inline uint32_t zig_fixunsdfsi(zig_f64 arg) { return __aeabi_d2uiz(arg); } zig_extern zig_callconv(pcs("aapcs")) uint64_t __aeabi_d2ulz(zig_f64); static inline uint64_t zig_fixunsdfdi(zig_f64 arg) { return __aeabi_d2ulz(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_i2d(int32_t); static inline zig_f64 zig_floatsidf(int32_t arg) { return __aeabi_i2d(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_ui2d(uint32_t); static inline zig_f64 zig_floatunsidf(uint32_t arg) { return __aeabi_ui2d(arg); } zig_extern zig_callconv(pcs("aapcs")) zig_f64 __aeabi_ul2d(uint64_t); static inline zig_f64 zig_floatundidf(uint64_t arg) { return __aeabi_ul2d(arg); } #else /* __ARM_EABI__ */ zig_float_builtins(32) zig_float_builtins(64) #endif /* __ARM_EABI__ */ /* ============================ Atomics Support ============================= */ /* Note that atomics should be implemented as macros because most compilers silently discard runtime atomic order information. */ /* Define fallback implementations first that can later be undef'd on compilers with builtin support. */ /* Note that zig_atomicrmw_expected is needed to handle aliasing between res and arg. */ #define zig_atomicrmw_xchg_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_min_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_max_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_xchg_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, arg, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_add_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_add_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_sub_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_sub_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_and_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_and_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_nand_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_not_##Type(zig_and_##Type(zig_atomicrmw_expected, arg), 128); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_or_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_or_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_xor_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_xor_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_min_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_min_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #define zig_atomicrmw_max_int128(res, obj, arg, order, Type, ReprType) do { \ zig_##Type zig_atomicrmw_expected; \ zig_##Type zig_atomicrmw_desired; \ zig_atomic_load(zig_atomicrmw_expected, obj, zig_memory_order_relaxed, Type, ReprType); \ do { \ zig_atomicrmw_desired = zig_max_##Type(zig_atomicrmw_expected, arg); \ } while (!zig_cmpxchg_weak(obj, zig_atomicrmw_expected, zig_atomicrmw_desired, order, zig_memory_order_relaxed, Type, ReprType)); \ res = zig_atomicrmw_expected; \ } while (0) #if (__STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__)) || (zig_has_include() && !defined(zig_msvc)) #define zig_c11_atomics #endif #if defined(zig_c11_atomics) #include typedef enum memory_order zig_memory_order; #define zig_memory_order_relaxed memory_order_relaxed #define zig_memory_order_acquire memory_order_acquire #define zig_memory_order_release memory_order_release #define zig_memory_order_acq_rel memory_order_acq_rel #define zig_memory_order_seq_cst memory_order_seq_cst #define zig_atomic(Type) _Atomic(Type) #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) #define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order) #define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order) #undef zig_atomicrmw_xchg_float #define zig_atomicrmw_xchg_float zig_atomicrmw_xchg #undef zig_atomicrmw_add_float #define zig_atomicrmw_add_float zig_atomicrmw_add #undef zig_atomicrmw_sub_float #define zig_atomicrmw_sub_float zig_atomicrmw_sub #elif defined(zig_gnuc) typedef int zig_memory_order; #define zig_memory_order_relaxed __ATOMIC_RELAXED #define zig_memory_order_acquire __ATOMIC_ACQUIRE #define zig_memory_order_release __ATOMIC_RELEASE #define zig_memory_order_acq_rel __ATOMIC_ACQ_REL #define zig_memory_order_seq_cst __ATOMIC_SEQ_CST #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, (ReprType *)&(expected), (ReprType *)&(desired), false, succ, fail) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, (ReprType *)&(expected), (ReprType *)&(desired), true, succ, fail) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, (ReprType *)&(arg), &(res), order) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) #define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, (ReprType *)&(arg), order) #define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order) #undef zig_atomicrmw_xchg_float #define zig_atomicrmw_xchg_float zig_atomicrmw_xchg #elif defined(zig_msvc) && defined(zig_x86) #define zig_memory_order_relaxed 0 #define zig_memory_order_acquire 2 #define zig_memory_order_release 3 #define zig_memory_order_acq_rel 4 #define zig_memory_order_seq_cst 5 #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_msvc_cmpxchg_##Type(obj, &(expected), desired) #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(obj, expected, desired, succ, fail, Type, ReprType) #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg) #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg) #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg) #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg) #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg) #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg) #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg) #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg) #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg) #define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg) #define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##order##_##Type(obj) /* TODO: zig_msvc && (zig_thumb || zig_aarch64) */ #else #define zig_memory_order_relaxed 0 #define zig_memory_order_acquire 2 #define zig_memory_order_release 3 #define zig_memory_order_acq_rel 4 #define zig_memory_order_seq_cst 5 #define zig_atomic(Type) Type #define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable #define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable #define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable #endif #if !defined(zig_c11_atomics) && defined(zig_msvc) && defined(zig_x86) /* TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64 */ #define zig_msvc_atomics(ZigType, Type, SigType, suffix, iso_suffix) \ static inline bool zig_msvc_cmpxchg_##ZigType(Type volatile* obj, Type* expected, Type desired) { \ Type comparand = *expected; \ Type initial = _InterlockedCompareExchange##suffix((SigType volatile*)obj, (SigType)desired, (SigType)comparand); \ bool exchanged = initial == comparand; \ if (!exchanged) { \ *expected = initial; \ } \ return exchanged; \ } \ static inline Type zig_msvc_atomicrmw_xchg_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedExchange##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_add_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedExchangeAdd##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_sub_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = prev - value; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_or_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedOr##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_xor_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedXor##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_and_##ZigType(Type volatile* obj, Type value) { \ return _InterlockedAnd##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomicrmw_nand_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = ~(prev & value); \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_min_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = value < prev ? value : prev; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline Type zig_msvc_atomicrmw_max_##ZigType(Type volatile* obj, Type value) { \ bool success = false; \ Type new; \ Type prev; \ while (!success) { \ prev = *obj; \ new = value > prev ? value : prev; \ success = zig_msvc_cmpxchg_##ZigType(obj, &prev, new); \ } \ return prev; \ } \ static inline void zig_msvc_atomic_store_##ZigType(Type volatile* obj, Type value) { \ (void)_InterlockedExchange##suffix((SigType volatile*)obj, (SigType)value); \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_relaxed_##ZigType(Type volatile* obj) { \ return __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_acquire_##ZigType(Type volatile* obj) { \ Type val = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ return val; \ } \ static inline Type zig_msvc_atomic_load_zig_memory_order_seq_cst_##ZigType(Type volatile* obj) { \ Type val = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ return val; \ } zig_msvc_atomics( u8, uint8_t, char, 8, 8) zig_msvc_atomics( i8, int8_t, char, 8, 8) zig_msvc_atomics(u16, uint16_t, short, 16, 16) zig_msvc_atomics(i16, int16_t, short, 16, 16) zig_msvc_atomics(u32, uint32_t, long, , 32) zig_msvc_atomics(i32, int32_t, long, , 32) #if defined(zig_x86_64) zig_msvc_atomics(u64, uint64_t, __int64, 64, 64) zig_msvc_atomics(i64, int64_t, __int64, 64, 64) #endif #define zig_msvc_flt_atomics(Type, SigType, suffix, iso_suffix) \ static inline bool zig_msvc_cmpxchg_##Type(zig_##Type volatile* obj, zig_##Type* expected, zig_##Type desired) { \ SigType exchange; \ SigType comparand; \ SigType initial; \ bool success; \ memcpy(&comparand, expected, sizeof(comparand)); \ memcpy(&exchange, &desired, sizeof(exchange)); \ initial = _InterlockedCompareExchange##suffix((SigType volatile*)obj, exchange, comparand); \ success = initial == comparand; \ if (!success) memcpy(expected, &initial, sizeof(*expected)); \ return success; \ } \ static inline void zig_msvc_atomic_store_##Type(zig_##Type volatile* obj, zig_##Type arg) { \ SigType value; \ memcpy(&value, &arg, sizeof(value)); \ (void)_InterlockedExchange##suffix((SigType volatile*)obj, value); \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_relaxed_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_acquire_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } \ static inline zig_##Type zig_msvc_atomic_load_zig_memory_order_seq_cst_##Type(zig_##Type volatile* obj) { \ zig_##Type result; \ SigType initial = __iso_volatile_load##iso_suffix((SigType volatile*)obj); \ _ReadWriteBarrier(); \ memcpy(&result, &initial, sizeof(result)); \ return result; \ } zig_msvc_flt_atomics(f32, long, , 32) #if defined(zig_x86_64) zig_msvc_flt_atomics(f64, int64_t, 64, 64) #endif #if defined(zig_x86_32) static inline void zig_msvc_atomic_barrier() { int32_t barrier; __asm { xchg barrier, eax } } static inline void* zig_msvc_atomicrmw_xchg_p32(void volatile* obj, void* arg) { return _InterlockedExchangePointer(obj, arg); } static inline void zig_msvc_atomic_store_p32(void volatile* obj, void* arg) { (void)_InterlockedExchangePointer(obj, arg); } static inline void* zig_msvc_atomic_load_zig_memory_order_relaxed_p32(void volatile* obj) { return (void*)__iso_volatile_load32(obj); } static inline void* zig_msvc_atomic_load_zig_memory_order_acquire_p32(void volatile* obj) { void* val = (void*)__iso_volatile_load32(obj); _ReadWriteBarrier(); return val; } static inline void* zig_msvc_atomic_load_zig_memory_order_seq_cst_p32(void volatile* obj) { return zig_msvc_atomic_load_zig_memory_order_acquire_p32(obj); } static inline bool zig_msvc_cmpxchg_p32(void volatile* obj, void* expected, void* desired) { void* comparand = *(void**)expected; void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand); bool success = initial == comparand; if (!success) *(void**)expected = initial; return success; } #else /* zig_x86_32 */ static inline void* zig_msvc_atomicrmw_xchg_p64(void volatile* obj, void* arg) { return _InterlockedExchangePointer(obj, arg); } static inline void zig_msvc_atomic_store_p64(void volatile* obj, void* arg) { (void)_InterlockedExchangePointer(obj, arg); } static inline void* zig_msvc_atomic_load_zig_memory_order_relaxed_p64(void volatile* obj) { return (void*)__iso_volatile_load64(obj); } static inline void* zig_msvc_atomic_load_zig_memory_order_acquire_p64(void volatile* obj) { void* val = (void*)__iso_volatile_load64(obj); _ReadWriteBarrier(); return val; } static inline void* zig_msvc_atomic_load_zig_memory_order_seq_cst_p64(void volatile* obj) { return zig_msvc_atomic_load_zig_memory_order_acquire_p64(obj); } static inline bool zig_msvc_cmpxchg_p64(void volatile* obj, void* expected, void* desired) { void* comparand = *(void**)expected; void* initial = _InterlockedCompareExchangePointer(obj, desired, comparand); bool success = initial == comparand; if (!success) *(void**)expected = initial; return success; } static inline bool zig_msvc_cmpxchg_u128(zig_u128 volatile* obj, zig_u128* expected, zig_u128 desired) { return _InterlockedCompareExchange128((__int64 volatile*)obj, (__int64)zig_hi_u128(desired), (__int64)zig_lo_u128(desired), (__int64*)expected); } static inline zig_u128 zig_msvc_atomic_load_u128(zig_u128 volatile* obj) { zig_u128 expected = zig_make_u128(UINT64_C(0), UINT64_C(0)); (void)zig_cmpxchg_strong(obj, expected, expected, zig_memory_order_seq_cst, zig_memory_order_seq_cst, u128, zig_u128); return expected; } static inline void zig_msvc_atomic_store_u128(zig_u128 volatile* obj, zig_u128 arg) { zig_u128 expected = zig_make_u128(UINT64_C(0), UINT64_C(0)); while (!zig_cmpxchg_weak(obj, expected, arg, zig_memory_order_seq_cst, zig_memory_order_seq_cst, u128, zig_u128)); } static inline bool zig_msvc_cmpxchg_i128(zig_i128 volatile* obj, zig_i128* expected, zig_i128 desired) { return _InterlockedCompareExchange128((__int64 volatile*)obj, (__int64)zig_hi_i128(desired), (__int64)zig_lo_i128(desired), (__int64*)expected); } static inline zig_i128 zig_msvc_atomic_load_i128(zig_i128 volatile* obj) { zig_i128 expected = zig_make_i128(INT64_C(0), UINT64_C(0)); (void)zig_cmpxchg_strong(obj, expected, expected, zig_memory_order_seq_cst, zig_memory_order_seq_cst, i128, zig_i128); return expected; } static inline void zig_msvc_atomic_store_i128(zig_i128 volatile* obj, zig_i128 arg) { zig_i128 expected = zig_make_i128(INT64_C(0), UINT64_C(0)); while (!zig_cmpxchg_weak(obj, expected, arg, zig_memory_order_seq_cst, zig_memory_order_seq_cst, i128, zig_i128)); } #endif /* zig_x86_32 */ #endif /* !zig_c11_atomics && zig_msvc && zig_x86 */ /* ======================== Special Case Intrinsics ========================= */ #if defined(zig_msvc) #include #endif #if defined(zig_thumb) static inline void* zig_thumb_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)_MoveFromCoprocessor(15, 0, 13, 0, 2); #elif defined(zig_gnuc_asm) __asm__ ("mrc p15, 0, %[ptr], c13, c0, 2" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_aarch64) static inline void* zig_aarch64_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readx18qword(0x0); #elif defined(zig_gnuc_asm) __asm__ ("mov %[ptr], x18" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_x86_32) static inline void* zig_x86_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readfsdword(0x18); #elif defined(zig_gnuc_asm) __asm__ ("movl %%fs:0x18, %[ptr]" : [ptr] "=r" (teb)); #endif return teb; } #elif defined(zig_x86_64) static inline void* zig_x86_64_windows_teb(void) { void* teb = 0; #if defined(zig_msvc) teb = (void*)__readgsqword(0x30); #elif defined(zig_gnuc_asm) __asm__ ("movq %%gs:0x30, %[ptr]" : [ptr] "=r" (teb)); #endif return teb; } #endif #if defined(zig_x86) static inline void zig_x86_cpuid(uint32_t leaf_id, uint32_t subid, uint32_t* eax, uint32_t* ebx, uint32_t* ecx, uint32_t* edx) { #if defined(zig_msvc) int cpu_info[4]; __cpuidex(cpu_info, leaf_id, subid); *eax = (uint32_t)cpu_info[0]; *ebx = (uint32_t)cpu_info[1]; *ecx = (uint32_t)cpu_info[2]; *edx = (uint32_t)cpu_info[3]; #elif defined(zig_gnuc_asm) __asm__("cpuid" : "=a"(*eax), "=b"(*ebx), "=c"(*ecx), "=d"(*edx) : "a"(leaf_id), "c"(subid)); #else *eax = 0; *ebx = 0; *ecx = 0; *edx = 0; #endif } static inline uint32_t zig_x86_get_xcr0(void) { #if defined(zig_msvc) return (uint32_t)_xgetbv(0); #elif defined(zig_gnuc_asm) uint32_t eax; uint32_t edx; __asm__("xgetbv" : "=a"(eax), "=d"(edx) : "c"(0)); return eax; #else *eax = 0; *ebx = 0; *ecx = 0; *edx = 0; #endif } #endif codspeed-4.4.1/instrument-hooks/scripts/release.py000064400000000000000000000010501046102023000204260ustar 00000000000000from pathlib import Path content = Path("zig-out/lib/core.c").read_text() print("Size of core.c:", len(content.splitlines())) valgrind_wrapper = Path("src/helpers/valgrind_wrapper.c").read_text() STUB = Path("scripts/stub.c").read_text() TEMPLATE = f"""\ // This file is generated by scripts/release.py // Do not edit it manually. // Include compatibility layer for cross-platform support #include "compat.h" #if defined(_WIN32) || defined(__APPLE__) {STUB} #else {content} #endif {valgrind_wrapper} """ Path("dist/core.c").write_text(TEMPLATE) codspeed-4.4.1/instrument-hooks/src/bincode.zig000064400000000000000000000442201046102023000176600ustar 00000000000000// Taken from here: https://github.com/qbradley/bincode-zig/blob/main/bincode.zig const std = @import("std"); const utils = @import("./utils.zig"); pub fn deserializeAlloc(stream: anytype, allocator: std.mem.Allocator, comptime T: type) !T { return switch (@typeInfo(T)) { .void => {}, .bool => try deserializeBool(stream), .float => try deserializeFloat(stream, T), .int => try deserializeInt(stream, T), .optional => |info| try deserializeOptionalAlloc(stream, allocator, info.child), .pointer => |info| try deserializePointerAlloc(stream, info, allocator), .array => |info| try deserializeArrayAlloc(stream, info, allocator), .@"struct" => |info| try deserializeStructAlloc(stream, info, allocator, T), .@"enum" => try deserializeEnum(stream, T), .@"union" => |info| try deserializeUnionAlloc(stream, info, allocator, T), else => unsupportedType(T), }; } pub fn deserialize(stream: anytype, comptime T: type) !T { return switch (@typeInfo(T)) { .void => {}, .bool => try deserializeBool(stream), .float => try deserializeFloat(stream, T), .int => try deserializeInt(stream, T), .optional => |info| try deserializeOptional(stream, info.child), .array => |info| try deserializeArray(stream, info), .@"struct" => |info| try deserializeStruct(stream, info, T), .@"enum" => try deserializeEnum(stream, T), .@"union" => |info| try deserializeUnion(stream, info, T), else => unsupportedType(T), }; } pub fn deserializeBuffer(comptime T: type, source: *[]const u8) T { return switch (@typeInfo(T)) { .void => {}, .bool => deserializeBufferBool(source), .float => deserializeBufferFloat(T, source), .int => deserializeBufferInt(T, source), .optional => |info| deserializeBufferOptional(info.child, source), .pointer => |info| deserializeBufferPointer(info, source), .array => |info| deserializeBufferArray(info, source), .@"struct" => |info| deserializeBufferStruct(T, info, source), .@"enum" => deserializeBufferEnum(T, source), .@"union" => |info| deserializeBufferUnion(T, info, source), else => unsupportedType(T), }; } pub fn serialize(stream: anytype, value: anytype) @TypeOf(stream).Error!void { const T = @TypeOf(value); return switch (@typeInfo(T)) { .void => {}, .bool => try serializeBool(stream, value), .float => try serializeFloat(stream, T, value), .int => try serializeInt(stream, T, value), .optional => |info| try serializeOptional(stream, info.child, value), .pointer => |info| try serializePointer(stream, info, T, value), .array => |info| try serializeArray(stream, info, T, value), .@"struct" => |info| try serializeStruct(stream, info, T, value), .@"enum" => try serializeEnum(stream, T, value), .@"union" => |info| try serializeUnion(stream, info, T, value), else => unsupportedType(T), }; } pub fn deserializeSliceIterator(comptime T: type, source: []const u8) DeserializeSliceIterator(T) { return DeserializeSliceIterator(T){ .source = source, }; } pub fn DeserializeSliceIterator(comptime T: type) type { return struct { source: []const u8, pub fn next(self: *@This()) ?T { if (self.source.len > 0) { return deserializeBuffer(T, &self.source); } else { return null; } } }; } fn deserializeBufferInt(comptime T: type, source_ptr: *[]const u8) T { const bytesRequired = @sizeOf(T); const source = source_ptr.*; if (bytesRequired <= source.len) { var tmp: [bytesRequired]u8 = undefined; std.mem.copy(u8, &tmp, source[0..bytesRequired]); source_ptr.* = source[bytesRequired..]; return std.mem.readIntLittle(T, &tmp); } else { invalidProtocol("Buffer ran out of bytes too soon."); } } fn deserializeBufferBool(source: *[]const u8) bool { return switch (deserializeBufferInt(u8, source)) { 0 => return false, 1 => return true, else => invalidProtocol("Boolean values should be encoded as a single byte with value 0 or 1 only."), }; } fn deserializeBufferOptional(comptime T: type, source: *[]const u8) ?T { if (deserializeBufferBool(source)) { return deserializeBuffer(T, source); } else { return null; } } fn deserializeBufferFloat(comptime T: type, source: *[]const u8) T { switch (T) { f32 => return @as(T, @bitCast(deserializeBufferInt(u32, source))), f64 => return @as(T, @bitCast(deserializeBufferInt(u64, source))), else => unsupportedType(T), } } fn deserializeBufferEnum(comptime T: type, source: *[]const u8) T { const raw_tag = deserializeBufferInt(u32, source); return @enumFromInt(raw_tag); } fn deserializeBufferStruct(comptime T: type, comptime info: std.builtin.Type.Struct, source: *[]const u8) T { var value: T = undefined; inline for (info.fields) |field| { @field(value, field.name) = deserializeBuffer(field.type, source); } return value; } fn deserializeBufferUnion(comptime T: type, comptime info: std.builtin.Type.Union, source: *[]const u8) T { if (info.tag_type) |Tag| { const raw_tag = deserializeBufferInt(u32, source); const tag = @as(T, @enumFromInt(raw_tag)); inline for (info.fields) |field| { if (tag == @field(Tag, field.name)) { const inner = deserializeBuffer(field.type, source); return @unionInit(T, field.name, inner); } } unreachable; } else { unsupportedType(T); } } fn deserializeBufferArray(comptime info: std.builtin.Type.Array, source_ptr: *[]const u8) [info.len]info.child { const T = @Type(.{ .array = info }); if (info.sentinel_ptr != null) unsupportedType(T); var value: T = undefined; if (info.child == u8) { const source = source_ptr.*; if (info.len <= source.len) { std.mem.copy(u8, &value, source[0..info.len]); source_ptr.* = source[info.len..]; } else { invalidProtocol("The stream end was found before all required bytes were read."); } } else { for (0..info.len) |idx| { value[idx] = deserializeBuffer(info.child, source_ptr); } } return value; } fn deserializeBufferPointer(comptime info: std.builtin.Type.Pointer, source_ptr: *[]const u8) []const info.child { const T = @Type(.{ .pointer = info }); if (info.sentinel_ptr != null) unsupportedType(T); switch (info.size) { .one => unsupportedType(T), .slice => { const len = @as(usize, @intCast(deserializeBufferInt(u64, source_ptr))); if (info.child == u8) { const source = source_ptr.*; if (len <= source.len) { source_ptr.* = source[len..]; return source[0..len]; } else { invalidProtocol("The stream end was found before all required bytes were read."); } } else { // we can't support a variable slice of types where the stream format // differs from in-memory format without allocating. unsupportedType(T); } }, .c => unsupportedType(T), .many => unsupportedType(T), } } fn deserializeBool(stream: anytype) !bool { switch (try stream.readInt(u8, .little)) { 0 => return false, 1 => return true, else => invalidProtocol("Boolean values should be encoded as a single byte with value 0 or 1 only."), } } fn deserializeFloat(stream: anytype, comptime T: type) !T { switch (T) { f32 => return @as(T, @bitCast(try stream.readInt(u32, .little))), f64 => return @as(T, @bitCast(try stream.readInt(u64, .little))), else => unsupportedType(T), } } fn deserializeInt(stream: anytype, comptime T: type) !T { switch (T) { i8 => return try stream.readInt(i8, .little), i16 => return try stream.readInt(i16, .little), i32 => return try stream.readInt(i32, .little), i64 => return try stream.readInt(i64, .little), i128 => return try stream.readInt(i128, .little), u8 => return try stream.readInt(u8, .little), u16 => return try stream.readInt(u16, .little), u32 => return try stream.readInt(u32, .little), u64 => return try stream.readInt(u64, .little), u128 => return try stream.readInt(u128, .little), else => unsupportedType(T), } } fn deserializeOptionalAlloc(stream: anytype, allocator: std.mem.Allocator, comptime T: type) !?T { switch (try stream.readInt(u8, .little)) { // None 0 => return null, // Some 1 => return try deserializeAlloc(stream, allocator, T), else => invalidProtocol("Optional is encoded as a single 0 valued byte for null, or a single 1 valued byte followed by the encoding of the contained value."), } } fn deserializeOptional(stream: anytype, comptime T: type) !?T { switch (try stream.readInt(u8, .little)) { // None 0 => return null, // Some 1 => return try deserialize(stream, T), else => invalidProtocol("Optional is encoded as a single 0 valued byte for null, or a single 1 valued byte followed by the encoding of the contained value."), } } fn deserializePointerAlloc(stream: anytype, comptime info: std.builtin.Type.Pointer, allocator: std.mem.Allocator) ![]info.child { const T = @Type(.{ .pointer = info }); if (info.sentinel_ptr != null) unsupportedType(T); switch (info.size) { .one => unsupportedType(T), .slice => { const len = @as(usize, @intCast(try stream.readInt(u64, .little))); var memory = try allocator.alloc(info.child, len); if (info.child == u8) { const amount = try stream.readAll(memory); if (amount != len) { invalidProtocol("The stream end was found before all required bytes were read."); } } else { for (0..len) |idx| { memory[idx] = try deserializeAlloc(stream, allocator, info.child); } } return memory; }, .c => unsupportedType(T), .many => unsupportedType(T), } } fn deserializeArrayAlloc(stream: anytype, comptime info: std.builtin.Type.Array, allocator: std.mem.Allocator) ![info.len]info.child { const T = @Type(.{ .array = info }); if (info.sentinel_ptr != null) unsupportedType(T); var value: T = undefined; if (info.child == u8) { const amount = try stream.readAll(value[0..]); if (amount != info.len) { invalidProtocol("The stream end was found before all required bytes were read."); } } else { for (0..info.len) |idx| { value[idx] = try deserializeAlloc(stream, allocator, info.child); } } return value; } fn deserializeArray(stream: anytype, comptime info: std.builtin.Type.Array) ![info.len]info.child { const T = @Type(.{ .array = info }); if (info.sentinel_ptr != null) unsupportedType(T); var value: T = undefined; if (info.child == u8) { const amount = try stream.readAll(value[0..]); if (amount != info.len) { invalidProtocol("The stream end was found before all required bytes were read."); } } else { for (0..info.len) |idx| { value[idx] = try deserialize(stream, info.child); } } return value; } fn deserializeStructAlloc(stream: anytype, comptime info: std.builtin.Type.Struct, allocator: std.mem.Allocator, comptime T: type) !T { var value: T = undefined; inline for (info.fields) |field| { @field(value, field.name) = try deserializeAlloc(stream, allocator, field.type); } return value; } fn deserializeStruct(stream: anytype, comptime info: std.builtin.Type.Struct, comptime T: type) !T { var value: T = undefined; inline for (info.fields) |field| { @field(value, field.name) = try deserialize(stream, field.type); } return value; } fn deserializeEnum(stream: anytype, comptime T: type) !T { const raw_tag = try deserializeInt(stream, u32); return @as(T, @enumFromInt(raw_tag)); } fn deserializeUnionAlloc(stream: anytype, comptime info: std.builtin.Type.Union, allocator: std.mem.Allocator, comptime T: type) !T { if (info.tag_type) |Tag| { const raw_tag = try deserializeAlloc(stream, allocator, u32); const tag = @as(Tag, @enumFromInt(raw_tag)); inline for (info.fields) |field| { if (tag == @field(Tag, field.name)) { const inner = try deserializeAlloc(stream, allocator, field.type); return @unionInit(T, field.name, inner); } } unreachable; } else { unsupportedType(T); } } fn deserializeUnion(stream: anytype, comptime info: std.builtin.Type.Union, comptime T: type) !T { if (info.tag_type) |Tag| { const raw_tag = try deserialize(stream, u32); const tag = @as(Tag, @enumFromInt(raw_tag)); inline for (info.fields) |field| { if (tag == @field(Tag, field.name)) { const inner = try deserialize(stream, field.type); return @unionInit(T, field.name, inner); } } unreachable; } else { unsupportedType(T); } } pub fn serializeBool(stream: anytype, value: bool) @TypeOf(stream).Error!void { const code: u8 = if (value) @as(u8, 1) else @as(u8, 0); return stream.writeIntLittle(u8, code); } pub fn serializeFloat(stream: anytype, comptime T: type, value: T) @TypeOf(stream).Error!void { switch (T) { f32 => try stream.writeIntLittle(u32, @as(u32, @bitCast(value))), f64 => try stream.writeIntLittle(u64, @as(u64, @bitCast(value))), else => unsupportedType(T), } } pub fn serializeInt(stream: anytype, comptime T: type, value: T) @TypeOf(stream).Error!void { switch (T) { i8 => try stream.writeInt(i8, value, .little), i16 => try stream.writeInt(i16, value, .little), i32 => try stream.writeInt(i32, value, .little), i64 => try stream.writeInt(i64, value, .little), i128 => try stream.writeInt(i128, value, .little), u8 => try stream.writeInt(u8, value, .little), u16 => try stream.writeInt(u16, value, .little), u32 => try stream.writeInt(u32, value, .little), u64 => try stream.writeInt(u64, value, .little), u128 => try stream.writeInt(u128, value, .little), else => unsupportedType(T), } } pub fn serializeOptional(stream: anytype, comptime T: type, value: ?T) @TypeOf(stream).Error!void { if (value) |actual| { try stream.writeInt(u8, 1, .little); try serialize(stream, actual); } else { // None try stream.writeInt(u8, 0, .little); } } pub fn serializePointer(stream: anytype, comptime info: std.builtin.Type.Pointer, comptime T: type, value: T) @TypeOf(stream).Error!void { if (info.sentinel_ptr != null) unsupportedType(T); switch (info.size) { .one => unsupportedType(T), .slice => { try stream.writeInt(u64, value.len, .little); if (info.child == u8) { try stream.writeAll(value); } else { for (value) |item| { try serialize(stream, item); } } }, .c => unsupportedType(T), .many => unsupportedType(T), } } pub fn serializeArray(stream: anytype, comptime info: std.builtin.Type.Array, comptime T: type, value: T) @TypeOf(stream).Error!void { if (info.sentinel_ptr != null) unsupportedType(T); if (info.child == u8) { try stream.writeAll(value); } else { for (value) |item| { try serialize(stream, item); } } } pub fn serializeStruct(stream: anytype, comptime info: std.builtin.Type.Struct, comptime T: type, value: T) @TypeOf(stream).Error!void { inline for (info.fields) |field| { try serialize(stream, @field(value, field.name)); } } pub fn serializeEnum(stream: anytype, comptime T: type, value: T) @TypeOf(stream).Error!void { const tag: u32 = @intFromEnum(value); try serialize(stream, tag); } pub fn serializeUnion(stream: anytype, comptime info: std.builtin.Type.Union, comptime T: type, value: T) @TypeOf(stream).Error!void { if (info.tag_type) |UnionTagType| { const tag: u32 = @intFromEnum(value); try serialize(stream, tag); inline for (info.fields) |field| { if (value == @field(UnionTagType, field.name)) { try serialize(stream, @field(value, field.name)); } } } else { unsupportedType(T); } } fn unsupportedType(comptime T: type) noreturn { @compileError("Unsupported type " ++ @typeName(T)); } fn invalidProtocol(comptime message: []const u8) noreturn { utils.print("Invalid protocol detected: %.*s\n", .{ @as(c_int, @intCast(message.len)), message.ptr }); std.process.exit(1); } test "example" { const bincode = @This(); //@import("bincode-zig"); const Shared = struct { name: []const u8, age: u32, }; const example = Shared{ .name = "Cat", .age = 5 }; // Serialize Shared to buffer var buffer: [8192]u8 = undefined; var output_stream = std.io.fixedBufferStream(buffer[0..]); try bincode.serialize(output_stream.writer(), example); // Use an arena to gather allocations from deserializer to make // them easy to clean up together. Allocations are required for // slices. var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); // Read what we wrote var input_stream = std.io.fixedBufferStream(output_stream.getWritten()); const copy = try bincode.deserializeAlloc( input_stream.reader(), arena.allocator(), Shared, ); // Make sure it is the same try std.testing.expectEqualStrings("Cat", copy.name); try std.testing.expectEqual(@as(u32, 5), copy.age); } codspeed-4.4.1/instrument-hooks/src/c.zig000064400000000000000000000076321046102023000165050ustar 00000000000000const instruments = @import("./instruments/root.zig"); const InstrumentHooks = instruments.InstrumentHooks; const builtin = @import("builtin"); const features = @import("./features.zig"); const shared = @import("./shared.zig"); const std = @import("std"); const utils = @import("./utils.zig"); pub const panic = if (builtin.is_test) std.debug.FullPanic(std.debug.defaultPanic) else std.debug.no_panic; const allocator = if (builtin.is_test) std.testing.allocator else std.heap.c_allocator; pub export fn instrument_hooks_set_feature(feature: u64, enabled: bool) void { const feature_enum = @as(features.Feature, @enumFromInt(feature)); features.set_feature(feature_enum, enabled); } pub export fn instrument_hooks_init() ?*InstrumentHooks { const hooks = allocator.create(InstrumentHooks) catch { return null; }; hooks.* = InstrumentHooks.init(allocator) catch { allocator.destroy(hooks); return null; }; return hooks; } pub export fn instrument_hooks_deinit(hooks: ?*InstrumentHooks) void { if (hooks) |h| { h.deinit(); allocator.destroy(h); } } pub export fn instrument_hooks_is_instrumented(hooks: ?*InstrumentHooks) bool { if (hooks) |h| { return h.is_instrumented(); } return false; } pub export fn instrument_hooks_start_benchmark(hooks: ?*InstrumentHooks) u8 { if (hooks) |h| { h.start_benchmark() catch { return 1; }; } return 0; } pub export fn instrument_hooks_stop_benchmark(hooks: ?*InstrumentHooks) u8 { if (hooks) |h| { h.stop_benchmark() catch { return 1; }; } return 0; } pub export fn instrument_hooks_set_executed_benchmark(hooks: ?*InstrumentHooks, pid: u32, uri: [*c]const u8) u8 { if (hooks) |h| { h.set_executed_benchmark(pid, uri) catch { return 1; }; } return 0; } // Deprecated: use instrument_hooks_set_executed_benchmark instead pub export fn instrument_hooks_executed_benchmark(hooks: ?*InstrumentHooks, pid: u32, uri: [*c]const u8) u8 { return instrument_hooks_set_executed_benchmark(hooks, pid, uri); } pub export fn instrument_hooks_set_integration(hooks: ?*InstrumentHooks, name: [*c]const u8, version: [*c]const u8) u8 { if (hooks) |h| { h.set_integration(name, version) catch { return 1; }; } return 0; } pub const MARKER_TYPE_SAMPLE_START: u8 = 0; pub const MARKER_TYPE_SAMPLE_END: u8 = 1; pub const MARKER_TYPE_BENCHMARK_START: u8 = 2; pub const MARKER_TYPE_BENCHMARK_END: u8 = 3; pub export fn instrument_hooks_add_marker(hooks: ?*InstrumentHooks, pid: u32, marker_type: u8, timestamp: u64) u8 { if (hooks) |h| { const marker_enum = switch (marker_type) { MARKER_TYPE_SAMPLE_START => shared.MarkerType{ .SampleStart = timestamp }, MARKER_TYPE_SAMPLE_END => shared.MarkerType{ .SampleEnd = timestamp }, MARKER_TYPE_BENCHMARK_START => shared.MarkerType{ .BenchmarkStart = timestamp }, MARKER_TYPE_BENCHMARK_END => shared.MarkerType{ .BenchmarkEnd = timestamp }, else => return 2, // Invalid marker type }; h.add_marker(pid, marker_enum) catch { return 1; }; } return 0; } pub export fn instrument_hooks_current_timestamp() u64 { return utils.clock_gettime_monotonic(); } test "no crash when not instrumented" { const instance = instrument_hooks_init(); defer instrument_hooks_deinit(instance); _ = instrument_hooks_is_instrumented(instance); _ = instrument_hooks_set_feature(0, true); try std.testing.expectEqual(0, instrument_hooks_start_benchmark(instance)); try std.testing.expectEqual(0, instrument_hooks_stop_benchmark(instance)); try std.testing.expectEqual(0, instrument_hooks_executed_benchmark(instance, 0, "test")); try std.testing.expectEqual(0, instrument_hooks_set_integration(instance, "pytest-codspeed", "1.0")); } codspeed-4.4.1/instrument-hooks/src/features.zig000064400000000000000000000020341046102023000200700ustar 00000000000000const std = @import("std"); pub const Feature = enum(u64) { // When enabled, we will not call the CALLGRIND_START_INSTRUMENTATION and CALLGRIND_STOP_INSTRUMENTATION markers. // This is useful when you notice additional overhead from this library in microbenchmarks, but means you // have to manually call them in your code. disable_callgrind_markers = 0, }; var features = std.StaticBitSet(64).initEmpty(); pub fn set_feature(feature: Feature, enabled: bool) void { if (enabled) { features.set(@intFromEnum(feature)); } else { features.unset(@intFromEnum(feature)); } } pub fn is_feature_enabled(feature: Feature) bool { return features.isSet(@intFromEnum(feature)); } test "set_feature and is_feature_enabled" { set_feature(Feature.disable_callgrind_markers, false); try std.testing.expect(!is_feature_enabled(Feature.disable_callgrind_markers)); set_feature(Feature.disable_callgrind_markers, true); try std.testing.expect(is_feature_enabled(Feature.disable_callgrind_markers)); } codspeed-4.4.1/instrument-hooks/src/fifo.zig000064400000000000000000000270621046102023000172050ustar 00000000000000const bincode = @import("bincode.zig"); const std = @import("std"); const shared = @import("shared.zig"); const fs = std.fs; const os = std.os; const mem = std.mem; const Allocator = std.mem.Allocator; const Path = []const u8; pub const Command = shared.Command; extern "c" fn mkfifo(path: [*:0]const u8, mode: c_uint) c_int; pub const UnixPipe = struct { pub const Reader = struct { file: fs.File, allocator: Allocator, buffer: std.ArrayList(u8), pub fn init(file: fs.File, allocator: Allocator) Reader { var buffer = std.ArrayList(u8).init(allocator); // Pre-allocate 1KB to avoid allocations for typical command sizes buffer.ensureTotalCapacity(1024) catch {}; return .{ .file = file, .allocator = allocator, .buffer = buffer, }; } pub fn read(self: *Reader, buffer: []u8) !usize { return self.file.read(buffer); } pub fn readAll(self: *Reader, buffer: []u8) !usize { return self.file.readAll(buffer); } // IMPORTANT: Caller is responsible for freeing the returned command. pub fn recvCmd(self: *Reader) !Command { // First read the length (u32 = 4 bytes) var len_buffer: [4]u8 = undefined; const len_read = self.file.readAll(&len_buffer) catch |err| { // Convert blocking/broken pipe errors to something the timeout logic can handle return switch (err) { error.WouldBlock, error.BrokenPipe => error.NotReady, else => err, }; }; if (len_read < 4) { return error.UnexpectedEof; } const message_len = std.mem.readInt(u32, &len_buffer, std.builtin.Endian.little); // Resize buffer to fit message (only allocates if growing) try self.buffer.resize(message_len); const msg_read = self.file.readAll(self.buffer.items) catch |err| { return switch (err) { error.WouldBlock, error.BrokenPipe => error.NotReady, else => err, }; }; if (msg_read < message_len) { return error.UnexpectedEof; } var stream = std.io.fixedBufferStream(self.buffer.items); return try bincode.deserializeAlloc(stream.reader(), self.allocator, Command); } pub fn waitForResponse(self: *Reader, timeout_ns: ?u64) !Command { const start = std.time.nanoTimestamp(); const timeout = timeout_ns orelse std.time.ns_per_s * 1; // Default 1 second timeout while (true) { const elapsed = @as(u64, @intCast(std.time.nanoTimestamp() - start)); if (elapsed > timeout) { return error.AckTimeout; } const cmd = self.recvCmd() catch |err| { // Only retry on transient errors, propagate fatal ones switch (err) { error.NotReady, error.UnexpectedEof => { const utils = @import("utils.zig"); utils.sleep(std.time.ns_per_ms * 10); continue; }, else => return err, } }; return cmd; } } pub fn waitForAck(self: *Reader, timeout_ns: ?u64) !void { const response = try self.waitForResponse(timeout_ns); defer response.deinit(self.allocator); switch (response) { .Ack => return, .Err => return error.UnexpectedError, else => { const logger = @import("logger.zig"); logger.debug("waitForAck received unexpected response: {}\n", .{response}); return error.UnexpectedResponse; }, } } pub fn deinit(self: *Reader) void { // Drain any pending data from the FIFO before closing to prevent // stale messages from being read by subsequent connections. // This is crucial when multiple instrument types probe the same FIFO // (e.g., AnalysisInstrument fails, then PerfInstrument tries). var dummy_buffer: [4096]u8 = undefined; while (true) { const bytes_read = self.file.read(&dummy_buffer) catch break; if (bytes_read == 0) break; } self.buffer.deinit(); self.file.close(); } }; pub const Writer = struct { file: fs.File, allocator: Allocator, buffer: std.ArrayList(u8), pub fn init(file: fs.File, allocator: Allocator) Writer { var buffer = std.ArrayList(u8).init(allocator); // Pre-allocate 1KB to avoid allocations for typical command sizes buffer.ensureTotalCapacity(1024) catch {}; return .{ .file = file, .allocator = allocator, .buffer = buffer, }; } pub fn write(self: *Writer, buffer: []const u8) !usize { return self.file.write(buffer); } pub fn writeAll(self: *Writer, buffer: []const u8) !void { return self.file.writeAll(buffer); } pub fn sendCmd(self: *Writer, cmd: Command) !void { // Clear buffer but keep allocated capacity self.buffer.clearRetainingCapacity(); try bincode.serialize(self.buffer.writer(), cmd); const bytes = self.buffer.items; try self.file.writeAll(std.mem.asBytes(&@as(u32, @intCast(bytes.len)))); try self.file.writeAll(bytes); } pub fn deinit(self: *Writer) void { self.buffer.deinit(); self.file.close(); } }; /// Create a new named pipe at the given path pub fn create(path: [*:0]const u8) !void { // Remove the previous FIFO (if it exists) fs.deleteFileAbsolute(std.mem.span(path)) catch {}; if (mkfifo(path, 0o700) != 0) { return error.FifoCreationFailed; } } fn openPipe(path: []const u8) !fs.File { try fs.accessAbsolute(path, .{ .mode = .read_write }); const file = try fs.openFileAbsolute(path, .{ .mode = .read_write, }); // Zig doesn't set the nonblocking flag correctly, so we have to do it manually. const utils = @import("utils.zig"); utils.setNonBlocking(file.handle); return file; } pub fn openRead(allocator: Allocator, path: []const u8) !Reader { const file = try openPipe(path); return Reader.init(file, allocator); } pub fn openWrite(allocator: Allocator, path: []const u8) !Writer { const file = try openPipe(path); return Writer.init(file, allocator); } }; pub fn sendCmd(allocator: Allocator, cmd: Command) !void { var writer = try UnixPipe.openWrite(allocator, shared.RUNNER_CTL_FIFO); defer writer.deinit(); try writer.sendCmd(cmd); var reader = try UnixPipe.openRead(allocator, shared.RUNNER_ACK_FIFO); defer reader.deinit(); try reader.waitForAck(null); } pub fn sendVersion(allocator: Allocator, protocol_version: u64) !void { const cmd = Command{ .SetVersion = protocol_version }; try sendCmd(allocator, cmd); } test "fail if doesn't exist" { const allocator = std.testing.allocator; const nonexistent_path = "/tmp/nonexistent_pipe_test.fifo"; // Ensure it doesn't exist fs.deleteFileAbsolute(nonexistent_path) catch {}; // Attempt to open for reading should fail const reader_result = UnixPipe.openRead(allocator, nonexistent_path); try std.testing.expectError(error.FileNotFound, reader_result); // Attempt to open for writing should fail const writer_result = UnixPipe.openWrite(allocator, nonexistent_path); try std.testing.expectError(error.FileNotFound, writer_result); // Attempt to send cmd to runner fifo fs.deleteFileAbsolute(shared.RUNNER_ACK_FIFO) catch {}; fs.deleteFileAbsolute(shared.RUNNER_CTL_FIFO) catch {}; const sendcmd_result = sendCmd(allocator, Command.StartBenchmark); try std.testing.expectError(error.FileNotFound, sendcmd_result); } test "unix pipe write read" { const allocator = std.testing.allocator; const test_path = "/tmp/test1.fifo"; try UnixPipe.create(test_path); var reader = try UnixPipe.openRead(allocator, test_path); defer reader.deinit(); var writer = try UnixPipe.openWrite(allocator, test_path); defer writer.deinit(); const message = "Hello"; try writer.writeAll(message); var buffer: [5]u8 = undefined; _ = try reader.readAll(&buffer); try std.testing.expectEqualStrings(message, &buffer); } test "unix pipe send recv cmd" { const allocator = std.testing.allocator; const test_path = "/tmp/test2.fifo"; try UnixPipe.create(test_path); var reader = try UnixPipe.openRead(allocator, test_path); defer reader.deinit(); var writer = try UnixPipe.openWrite(allocator, test_path); defer writer.deinit(); try writer.sendCmd(Command.StartBenchmark); const cmd = try reader.recvCmd(); defer cmd.deinit(writer.allocator); try std.testing.expectEqual(Command.StartBenchmark, cmd); } test "unix pipe send without ack" { const allocator = std.testing.allocator; const test_path = "/tmp/test_no_ack.fifo"; try UnixPipe.create(test_path); // Open both reader and writer so they don't block on open var reader = try UnixPipe.openRead(allocator, test_path); defer reader.deinit(); var writer = try UnixPipe.openWrite(allocator, test_path); defer writer.deinit(); // Writer doesn't send anything, so waitForResponse should timeout const result = reader.waitForResponse(std.time.ns_per_ms * 100); try std.testing.expectError(error.AckTimeout, result); } test "unix pipe prevents stale messages between connections" { const allocator = std.testing.allocator; const test_path = "/tmp/test_stale_messages.fifo"; try UnixPipe.create(test_path); // Keep writer open throughout to maintain the FIFO var writer = try UnixPipe.openWrite(allocator, test_path); defer writer.deinit(); // STEP 1: Simulate first connection { var first_reader = try UnixPipe.openRead(allocator, test_path); // Send and successfully read first command try writer.sendCmd(Command.StartBenchmark); const cmd1 = try first_reader.recvCmd(); defer cmd1.deinit(allocator); try std.testing.expect(cmd1.equal(Command.StartBenchmark)); // Send second command but DON'T read it try writer.sendCmd(Command.StopBenchmark); // Close first reader WITHOUT reading the second command // This should drain the unread StopBenchmark message first_reader.deinit(); } // STEP 2: Simulate second connection { var second_reader = try UnixPipe.openRead(allocator, test_path); defer second_reader.deinit(); // Send fresh command try writer.sendCmd(Command.Ack); // This should read the fresh Ack, NOT the stale StopBenchmark const cmd2 = try second_reader.recvCmd(); defer cmd2.deinit(allocator); // CRITICAL ASSERTION: We should receive the fresh Ack // Without the drain logic, this would fail with StopBenchmark try std.testing.expect(cmd2.equal(Command.Ack)); } } codspeed-4.4.1/instrument-hooks/src/helpers/valgrind.zig000064400000000000000000000006351046102023000215270ustar 00000000000000const builtin = @import("builtin"); const std = @import("std"); const math = std.math; pub extern "C" fn running_on_valgrind() u8; pub extern "C" fn callgrind_dump_stats() void; pub extern "C" fn callgrind_dump_stats_at(pos_str: [*:0]const u8) void; pub extern "C" fn callgrind_zero_stats() void; pub extern "C" fn callgrind_start_instrumentation() void; pub extern "C" fn callgrind_stop_instrumentation() void; codspeed-4.4.1/instrument-hooks/src/instruments/analysis.zig000064400000000000000000000003441046102023000224720ustar 00000000000000const shared = @import("../shared.zig"); const fifo_instrument = @import("fifo_instrument.zig"); const AnalysisError = error{ModeError}; pub const AnalysisInstrument = fifo_instrument.FifoInstrument(.Analysis, AnalysisError); codspeed-4.4.1/instrument-hooks/src/instruments/fifo_instrument.zig000064400000000000000000000037571046102023000240750ustar 00000000000000const std = @import("std"); const runner_fifo = @import("../runner_fifo.zig"); const shared = @import("../shared.zig"); /// Creates a complete FIFO-based instrument struct that validates a specific integration mode pub fn FifoInstrument(comptime mode: shared.IntegrationMode, comptime error_type: anytype) type { return struct { fifo: runner_fifo.RunnerFifo, const Self = @This(); pub fn init(allocator: std.mem.Allocator) !Self { var fifo = try runner_fifo.RunnerFifo.init(allocator); // Ensure both the runner and integration FIFO are compatible try fifo.validate_protocol_version(); // Get the instrumentation mode from the runner const detected_mode = fifo.get_integration_mode() catch |err| { fifo.deinit(); return err; }; // Only accept if the runner is in the correct mode if (detected_mode != mode) { fifo.deinit(); return error_type.ModeError; } return Self{ .fifo = fifo }; } pub fn deinit(self: *Self) void { self.fifo.deinit(); } pub fn send_version(self: *Self, protocol_version: u64) !void { try self.fifo.send_version(protocol_version); } pub fn start_benchmark(self: *Self) !void { try self.fifo.start_benchmark(); } pub fn stop_benchmark(self: *Self) !void { try self.fifo.stop_benchmark(); } pub fn set_executed_benchmark(self: *Self, pid: u32, uri: [*c]const u8) !void { try self.fifo.set_executed_benchmark(pid, uri); } pub fn set_integration(self: *Self, name: [*c]const u8, version: [*c]const u8) !void { try self.fifo.set_integration(name, version); } pub fn add_marker(self: *Self, pid: u32, marker: shared.MarkerType) !void { try self.fifo.add_marker(pid, marker); } }; } codspeed-4.4.1/instrument-hooks/src/instruments/perf.zig000064400000000000000000000003241046102023000216010ustar 00000000000000const shared = @import("../shared.zig"); const fifo_instrument = @import("fifo_instrument.zig"); const PerfError = error{ModeError}; pub const PerfInstrument = fifo_instrument.FifoInstrument(.Perf, PerfError); codspeed-4.4.1/instrument-hooks/src/instruments/root.zig000064400000000000000000000062741046102023000216420ustar 00000000000000const std = @import("std"); const perf = @import("perf.zig"); const analysis = @import("analysis.zig"); const valgrind = @import("valgrind.zig"); const shared = @import("../shared.zig"); const ValgrindInstrument = valgrind.ValgrindInstrument; const PerfInstrument = perf.PerfInstrument; const AnalysisInstrument = analysis.AnalysisInstrument; pub const InstrumentHooks = union(enum) { valgrind: ValgrindInstrument, perf: PerfInstrument, analysis: AnalysisInstrument, none: void, const Self = @This(); pub fn init(allocator: std.mem.Allocator) !Self { if (ValgrindInstrument.init(allocator)) |valgrind_inst| { return Self{ .valgrind = valgrind_inst }; } else |_| {} if (AnalysisInstrument.init(allocator)) |analysis_inst| { return Self{ .analysis = analysis_inst }; } else |_| {} if (PerfInstrument.init(allocator)) |perf_inst| { return Self{ .perf = perf_inst }; } else |_| {} return Self{ .none = {} }; } pub inline fn deinit(self: *Self) void { switch (self.*) { .valgrind => {}, .perf => self.perf.deinit(), .analysis => self.analysis.deinit(), .none => {}, } } pub inline fn is_instrumented(self: *Self) bool { return switch (self.*) { .valgrind => ValgrindInstrument.is_instrumented(), .perf => true, .analysis => true, .none => false, }; } pub inline fn start_benchmark(self: *Self) !void { if (self.* == .perf) { return self.perf.start_benchmark(); } else if (self.* == .valgrind) { return ValgrindInstrument.start_benchmark(); } else if (self.* == .analysis) { return self.analysis.start_benchmark(); } } pub inline fn stop_benchmark(self: *Self) !void { if (self.* == .valgrind) { return ValgrindInstrument.stop_benchmark(); } else if (self.* == .perf) { return self.perf.stop_benchmark(); } else if (self.* == .analysis) { return self.analysis.stop_benchmark(); } } pub inline fn set_executed_benchmark(self: *Self, pid: u32, uri: [*c]const u8) !void { switch (self.*) { .valgrind => ValgrindInstrument.set_executed_benchmark(pid, uri), .perf => try self.perf.set_executed_benchmark(pid, uri), .analysis => try self.analysis.set_executed_benchmark(pid, uri), .none => {}, } } pub inline fn set_integration(self: *Self, name: [*c]const u8, version: [*c]const u8) !void { switch (self.*) { .valgrind => try self.valgrind.set_integration(name, version), .perf => try self.perf.set_integration(name, version), .analysis => try self.analysis.set_integration(name, version), .none => {}, } } pub inline fn add_marker(self: *Self, pid: u32, marker: shared.MarkerType) !void { if (self.* == .perf) { return self.perf.add_marker(pid, marker); } else if (self.* == .analysis) { return self.analysis.add_marker(pid, marker); } } }; codspeed-4.4.1/instrument-hooks/src/instruments/valgrind.zig000064400000000000000000000027251046102023000224620ustar 00000000000000const std = @import("std"); const valgrind = @import("../helpers/valgrind.zig"); const features = @import("../features.zig"); pub const ValgrindInstrument = struct { allocator: std.mem.Allocator, const Self = @This(); pub fn init(allocator: std.mem.Allocator) !Self { if (!ValgrindInstrument.is_instrumented()) { return error.NotInstrumented; } return Self{ .allocator = allocator, }; } pub inline fn is_instrumented() bool { return valgrind.running_on_valgrind() > 0; } pub inline fn set_integration(self: Self, name: [*c]const u8, version: [*c]const u8) !void { const metadata = try std.fmt.allocPrintZ( self.allocator, "Metadata: {s} {s}", .{ name, version }, ); defer self.allocator.free(metadata); valgrind.callgrind_dump_stats_at(metadata.ptr); } pub inline fn start_benchmark() void { if (!features.is_feature_enabled(.disable_callgrind_markers)) { valgrind.callgrind_zero_stats(); valgrind.callgrind_start_instrumentation(); } } pub inline fn stop_benchmark() void { if (!features.is_feature_enabled(.disable_callgrind_markers)) { valgrind.callgrind_stop_instrumentation(); } } pub inline fn set_executed_benchmark(pid: u32, uri: [*c]const u8) void { _ = pid; valgrind.callgrind_dump_stats_at(uri); } }; codspeed-4.4.1/instrument-hooks/src/logger.zig000064400000000000000000000030501046102023000175300ustar 00000000000000const std = @import("std"); // We have to use c_char here, otherwise we redeclare it and cast u8 to char which results in // additional warnings (-Wbuiltin-declaration-mismatch and -Wpointer-sign). extern "c" fn printf(format: [*c]const c_char, ...) c_int; pub const LogLevel = enum(u8) { Debug = 0, Info = 1, Warn = 2, Error = 3, }; var max_level: LogLevel = LogLevel.Debug; fn logWithPrefix(comptime level: LogLevel, comptime fmt: []const u8, args: anytype) void { if (@intFromEnum(level) < @intFromEnum(max_level)) { return; } // Format the message using Zig's formatting var buffer: [512]u8 = undefined; const prefix_fmt = comptime switch (level) { .Debug => "[DEBUG] " ++ fmt, .Info => "[INFO] " ++ fmt, .Warn => "[WARN] " ++ fmt, .Error => "[ERROR] " ++ fmt, }; const msg = std.fmt.bufPrint(&buffer, prefix_fmt, args) catch { _ = printf(@as([*c]const c_char, @ptrCast("[ERROR] logger formatting failed\n"))); return; }; // Print the formatted message (printf with only format string, no args) _ = printf(@as([*c]const c_char, @ptrCast(msg.ptr))); } pub fn debug(comptime fmt: []const u8, args: anytype) void { logWithPrefix(.Debug, fmt, args); } pub fn info(comptime fmt: []const u8, args: anytype) void { logWithPrefix(.Info, fmt, args); } pub fn warn(comptime fmt: []const u8, args: anytype) void { logWithPrefix(.Warn, fmt, args); } pub fn err(comptime fmt: []const u8, args: anytype) void { logWithPrefix(.Error, fmt, args); } codspeed-4.4.1/instrument-hooks/src/root.zig000064400000000000000000000003101046102023000172300ustar 00000000000000test "run all tests" { _ = @import("tests/deserialize_rust/rust_deser.zig"); _ = @import("bincode.zig"); _ = @import("fifo.zig"); _ = @import("c.zig"); _ = @import("utils.zig"); } codspeed-4.4.1/instrument-hooks/src/runner_fifo.zig000064400000000000000000000155371046102023000206020ustar 00000000000000const std = @import("std"); const fifo = @import("fifo.zig"); const shared = @import("shared.zig"); const logger = @import("logger.zig"); // v1: Initial release // v2: Added GetIntegrationMode pub const PROTOCOL_VERSION: u64 = 2; pub const RunnerFifo = struct { allocator: std.mem.Allocator, writer: fifo.UnixPipe.Writer, reader: fifo.UnixPipe.Reader, const Self = @This(); pub fn init(allocator: std.mem.Allocator) !Self { return .{ .allocator = allocator, .writer = try fifo.UnixPipe.openWrite(allocator, shared.RUNNER_CTL_FIFO), .reader = try fifo.UnixPipe.openRead(allocator, shared.RUNNER_ACK_FIFO), }; } pub fn validate_protocol_version(self: *Self) !void { self.send_version(PROTOCOL_VERSION) catch |err| { switch (err) { // No runner present - silently continue as NOP error.AckTimeout => return, // Runner explicitly rejected version error.UnexpectedError => { logger.err("instrument-hooks: CodSpeed runner rejected protocol version {}\n", .{PROTOCOL_VERSION}); logger.err("instrument-hooks: please update the CodSpeed action to the latest version\n", .{}); std.posix.exit(1); }, // All other errors - log and exit else => { logger.err("instrument-hooks: error {s} during version check\n", .{@errorName(err)}); std.posix.exit(1); }, } }; } pub fn deinit(self: *Self) void { self.writer.deinit(); self.reader.deinit(); } pub fn send_cmd(self: *Self, cmd: fifo.Command) !void { try self.writer.sendCmd(cmd); try self.reader.waitForAck(null); } pub fn ping_perf(self: *Self) bool { self.send_cmd(fifo.Command.PingPerf) catch { return false; }; return true; } pub noinline fn start_benchmark(self: *Self) !void { @branchHint(.cold); // Prevent inline try self.writer.sendCmd(fifo.Command.StartBenchmark); try self.reader.waitForAck(null); } pub noinline fn stop_benchmark(self: *Self) !void { @branchHint(.cold); // Prevent inline try self.writer.sendCmd(fifo.Command.StopBenchmark); try self.reader.waitForAck(null); } pub fn set_executed_benchmark(self: *Self, pid: u32, uri: [*c]const u8) !void { try self.writer.sendCmd(fifo.Command{ .ExecutedBenchmark = .{ .pid = pid, .uri = std.mem.span(uri), } }); try self.reader.waitForAck(null); } pub fn set_integration(self: *Self, name: [*c]const u8, version: [*c]const u8) !void { try self.writer.sendCmd(fifo.Command{ .SetIntegration = .{ .name = std.mem.span(name), .version = std.mem.span(version), } }); try self.reader.waitForAck(null); } pub fn add_marker(self: *Self, pid: u32, marker: shared.MarkerType) !void { try self.writer.sendCmd(fifo.Command{ .AddMarker = .{ .pid = pid, .marker = marker, } }); try self.reader.waitForAck(null); } pub fn send_version(self: *Self, protocol_version: u64) !void { try self.writer.sendCmd(fifo.Command{ .SetVersion = protocol_version }); try self.reader.waitForAck(null); } pub fn get_integration_mode(self: *Self) !shared.IntegrationMode { // NOTE: Other messages send data to the runner, and expect an ACK response (see `sendCmd`). This // command expects the runner to respond with data, so have to write and read directly. try self.writer.sendCmd(fifo.Command.GetIntegrationMode); const response = try self.reader.waitForResponse(null); defer response.deinit(self.allocator); if (response == .IntegrationModeResponse) { return response.IntegrationModeResponse; } return error.UnexpectedResponse; } }; test "test runner fifo" { const allocator = std.testing.allocator; try fifo.UnixPipe.create(shared.RUNNER_ACK_FIFO); try fifo.UnixPipe.create(shared.RUNNER_CTL_FIFO); var ctl_fifo = try fifo.UnixPipe.openRead(allocator, shared.RUNNER_CTL_FIFO); defer ctl_fifo.deinit(); var ack_fifo = try fifo.UnixPipe.openWrite(allocator, shared.RUNNER_ACK_FIFO); defer ack_fifo.deinit(); const FifoTester = struct { allocator: std.mem.Allocator, ctl_pipe: *fifo.UnixPipe.Reader, ack_pipe: *fifo.UnixPipe.Writer, received_cmd: ?fifo.Command = null, error_occurred: bool = false, pub fn func(ctx: *@This()) void { const received_cmd = ctx.ctl_pipe.recvCmd() catch |err| { std.debug.print("Failed to receive command: {}\n", .{err}); ctx.error_occurred = true; return; }; ctx.received_cmd = received_cmd; ctx.ack_pipe.sendCmd(fifo.Command.Ack) catch |err| { std.debug.print("Failed to send ACK: {}\n", .{err}); ctx.error_occurred = true; }; } pub fn send(self: *@This(), comptime f: anytype, args: anytype) !fifo.Command { // 1. Create the thread which handles the command // 2. Execute the callback // 3. Wait for the thread to finish // const receiver_thread = try std.Thread.spawn(.{}, @This().func, .{self}); try @call(.auto, f, args); receiver_thread.join(); if (self.error_occurred) { return error.IntegrationError; } self.error_occurred = false; return self.received_cmd.?; } }; var tester = FifoTester{ .allocator = allocator, .ctl_pipe = &ctl_fifo, .ack_pipe = &ack_fifo, }; var runner_fifo = try RunnerFifo.init(allocator); defer runner_fifo.deinit(); const si_result = try tester.send(RunnerFifo.set_integration, .{ &runner_fifo, "zig", "0.10.0" }); try std.testing.expect(si_result.equal(fifo.Command{ .SetIntegration = .{ .name = "zig", .version = "0.10.0" } })); si_result.deinit(allocator); const cb_result = try tester.send(RunnerFifo.set_executed_benchmark, .{ &runner_fifo, 42, "foo" }); try std.testing.expect(cb_result.equal(fifo.Command{ .ExecutedBenchmark = .{ .pid = 42, .uri = "foo" } })); cb_result.deinit(allocator); const start_result = try tester.send(RunnerFifo.start_benchmark, .{&runner_fifo}); try std.testing.expect(start_result.equal(fifo.Command.StartBenchmark)); start_result.deinit(allocator); const stop_result = try tester.send(RunnerFifo.stop_benchmark, .{&runner_fifo}); try std.testing.expect(stop_result.equal(fifo.Command.StopBenchmark)); stop_result.deinit(allocator); } codspeed-4.4.1/instrument-hooks/src/shared.zig000064400000000000000000000125631046102023000175300ustar 00000000000000const std = @import("std"); // WARNING: Has to be in sync with `runner` pub const RUNNER_CTL_FIFO = "/tmp/runner.ctl.fifo"; pub const RUNNER_ACK_FIFO = "/tmp/runner.ack.fifo"; // The different markers that can be set in the perf.data. // // `SampleStart/End`: Marks the start and end of a sampling period. This is used to differentiate between benchmarks. // `BenchmarkStart/End`: Marks the start and end of a benchmark. This is used to measure the duration of a benchmark, without the benchmark harness code. pub const MarkerType = union(enum) { SampleStart: u64, SampleEnd: u64, BenchmarkStart: u64, BenchmarkEnd: u64, pub fn format( self: MarkerType, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = fmt; _ = options; switch (self) { .SampleStart => |ts| try writer.print("SampleStart({d})", .{ts}), .SampleEnd => |ts| try writer.print("SampleEnd({d})", .{ts}), .BenchmarkStart => |ts| try writer.print("BenchmarkStart({d})", .{ts}), .BenchmarkEnd => |ts| try writer.print("BenchmarkEnd({d})", .{ts}), } } pub fn equal(self: MarkerType, other: MarkerType) bool { return std.meta.eql(self, other); } }; pub const IntegrationMode = enum { Perf, Simulation, Analysis, }; pub const Command = union(enum) { ExecutedBenchmark: struct { pid: u32, uri: []const u8, }, StartBenchmark, StopBenchmark, Ack, PingPerf, SetIntegration: struct { name: []const u8, version: []const u8, }, Err, AddMarker: struct { pid: u32, marker: MarkerType, }, SetVersion: u64, GetIntegrationMode, IntegrationModeResponse: IntegrationMode, pub fn deinit(self: Command, allocator: std.mem.Allocator) void { switch (self) { .SetIntegration => |data| { allocator.free(data.name); allocator.free(data.version); }, .ExecutedBenchmark => |data| allocator.free(data.uri), .SetVersion => {}, .GetIntegrationMode => {}, .IntegrationModeResponse => {}, else => {}, } } pub fn format( self: Command, comptime fmt: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = fmt; _ = options; switch (self) { .ExecutedBenchmark => |data| try writer.print("ExecutedBenchmark {{ pid: {d}, uri: {s} }}", .{ data.pid, data.uri }), .StartBenchmark => try writer.writeAll("StartBenchmark"), .StopBenchmark => try writer.writeAll("StopBenchmark"), .Ack => try writer.writeAll("Ack"), .PingPerf => try writer.writeAll("PingPerf"), .SetIntegration => |data| try writer.print("SetIntegration {{ name: {s}, version: {s} }}", .{ data.name, data.version }), .Err => try writer.writeAll("Err"), .AddMarker => |data| try writer.print("AddMarker {{ pid: {d}, marker: {} }}", .{ data.pid, data.marker }), .SetVersion => |data| try writer.print("SetVersion {{ protocol_version: {d} }}", .{data}), .GetIntegrationMode => try writer.writeAll("GetIntegrationMode"), .IntegrationModeResponse => |mode| try writer.print("IntegrationModeResponse {}", .{mode}), } } pub fn equal(self: Command, other: Command) bool { return switch (self) { .ExecutedBenchmark => |self_data| switch (other) { .ExecutedBenchmark => |other_data| self_data.pid == other_data.pid and std.mem.eql(u8, self_data.uri, other_data.uri), else => false, }, .StartBenchmark => switch (other) { .StartBenchmark => true, else => false, }, .StopBenchmark => switch (other) { .StopBenchmark => true, else => false, }, .Ack => switch (other) { .Ack => true, else => false, }, .PingPerf => switch (other) { .PingPerf => true, else => false, }, .SetIntegration => |self_data| switch (other) { .SetIntegration => |other_data| std.mem.eql(u8, self_data.name, other_data.name) and std.mem.eql(u8, self_data.version, other_data.version), else => false, }, .Err => switch (other) { .Err => true, else => false, }, .AddMarker => |self_data| switch (other) { .AddMarker => |other_data| self_data.pid == other_data.pid and self_data.marker.equal(other_data.marker), else => false, }, .SetVersion => |self_data| switch (other) { .SetVersion => |other_data| self_data == other_data, else => false, }, .GetIntegrationMode => switch (other) { .GetIntegrationMode => true, else => false, }, .IntegrationModeResponse => |self_mode| switch (other) { .IntegrationModeResponse => |other_mode| self_mode == other_mode, else => false, }, }; } }; codspeed-4.4.1/instrument-hooks/src/tests/deserialize_rust/create_serialized.rs000075500000000000000000000052511046102023000263110ustar 00000000000000#!/usr/bin/env rust-script //! Serializes the shared `Command` enum to test whether Zig deserializes it correctly. //! //! # Run //! //! ```bash //! ./create_serialized.rs > serialized.zig //! ``` //! //! ```cargo //! [dependencies] //! bincode = "1.3.3" //! serde = { version = "1.0.192", features = ["derive"] } //! runner-shared = { git = "https://github.com/CodSpeedHQ/runner" } //! ``` // Import from runner-shared to ensure we use the same types use runner_shared::fifo::{Command, MarkerType, RunnerMode}; fn dump(name: &str, result: &Vec) { print!("pub const {}: []const u8 = &.{{ ", name); for byte in result.iter() { print!("0x{:X}, ", byte); } println!(" }};"); } fn example(name: &str, value: &T) { let result = bincode::serialize(&value).unwrap(); dump(name, &result); } fn main() { println!("// This file is generated using 'cargo run > rust_ser.zig'"); println!(""); example( "cmd_cur_bench", &Command::CurrentBenchmark { pid: 12345, uri: "http://example.com/benchmark".to_string(), }, ); example("cmd_start_bench", &Command::StartBenchmark); example("cmd_stop_bench", &Command::StopBenchmark); example("cmd_ack", &Command::Ack); example("cmd_ping_perf", &Command::PingPerf); example( "cmd_set_integration", &Command::SetIntegration { name: "test-integration".to_string(), version: "1.0.0".to_string(), }, ); example("cmd_err", &Command::Err); example( "cmd_add_marker_sample_start", &Command::AddMarker { pid: 12345, marker: MarkerType::SampleStart(1000), }, ); example( "cmd_add_marker_sample_end", &Command::AddMarker { pid: 12345, marker: MarkerType::SampleEnd(2000), }, ); example( "cmd_add_marker_benchmark_start", &Command::AddMarker { pid: 12345, marker: MarkerType::BenchmarkStart(3000), }, ); example( "cmd_add_marker_benchmark_end", &Command::AddMarker { pid: 12345, marker: MarkerType::BenchmarkEnd(4000), }, ); example("cmd_set_version", &Command::SetVersion(1)); example("cmd_get_runner_mode", &Command::GetRunnerMode); example( "cmd_runner_mode_perf", &Command::RunnerModeResponse(RunnerMode::Perf), ); example( "cmd_runner_mode_simulation", &Command::RunnerModeResponse(RunnerMode::Simulation), ); example( "cmd_runner_mode_analysis", &Command::RunnerModeResponse(RunnerMode::Analysis), ); } codspeed-4.4.1/instrument-hooks/src/tests/deserialize_rust/rust_deser.zig000064400000000000000000000046421046102023000251570ustar 00000000000000const std = @import("std"); const shared = @import("../../shared.zig"); const Command = shared.Command; fn assert_eq(serialized: []const u8, expected_cmd: Command) !void { const bincode = @import("../../bincode.zig"); var stream = std.io.fixedBufferStream(serialized); const deserialized_cmd = try bincode.deserializeAlloc(stream.reader(), std.testing.allocator, Command); defer deserialized_cmd.deinit(std.testing.allocator); try std.testing.expect(expected_cmd.equal(deserialized_cmd)); } test "rust deserialization" { const rust = @import("serialized.zig"); try assert_eq(rust.cmd_cur_bench, Command{ .ExecutedBenchmark = .{ .pid = 12345, .uri = "http://example.com/benchmark", } }); try assert_eq(rust.cmd_start_bench, Command{ .StartBenchmark = {} }); try assert_eq(rust.cmd_stop_bench, Command{ .StopBenchmark = {} }); try assert_eq(rust.cmd_ack, Command{ .Ack = {} }); try assert_eq(rust.cmd_ping_perf, Command{ .PingPerf = {} }); try assert_eq(rust.cmd_set_integration, Command{ .SetIntegration = .{ .name = "test-integration", .version = "1.0.0", } }); try assert_eq(rust.cmd_err, Command{ .Err = {} }); // AddMarker commands try assert_eq(rust.cmd_add_marker_sample_start, Command{ .AddMarker = .{ .pid = 12345, .marker = shared.MarkerType{ .SampleStart = 1000 }, } }); try assert_eq(rust.cmd_add_marker_sample_end, Command{ .AddMarker = .{ .pid = 12345, .marker = shared.MarkerType{ .SampleEnd = 2000 }, } }); try assert_eq(rust.cmd_add_marker_benchmark_start, Command{ .AddMarker = .{ .pid = 12345, .marker = shared.MarkerType{ .BenchmarkStart = 3000 }, } }); try assert_eq(rust.cmd_add_marker_benchmark_end, Command{ .AddMarker = .{ .pid = 12345, .marker = shared.MarkerType{ .BenchmarkEnd = 4000 }, } }); // SetVersion command try assert_eq(rust.cmd_set_version, Command{ .SetVersion = 1 }); // GetIntegrationMode command try assert_eq(rust.cmd_get_runner_mode, Command{ .GetIntegrationMode = {} }); // IntegrationModeResponse commands try assert_eq(rust.cmd_runner_mode_perf, Command{ .IntegrationModeResponse = .Perf }); try assert_eq(rust.cmd_runner_mode_simulation, Command{ .IntegrationModeResponse = .Simulation }); try assert_eq(rust.cmd_runner_mode_analysis, Command{ .IntegrationModeResponse = .Analysis }); } codspeed-4.4.1/instrument-hooks/src/tests/deserialize_rust/serialized.zig000064400000000000000000000056241046102023000251340ustar 00000000000000// This file is generated using 'cargo run > rust_ser.zig' pub const cmd_cur_bench: []const u8 = &.{ 0x0, 0x0, 0x0, 0x0, 0x39, 0x30, 0x0, 0x0, 0x1C, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x68, 0x74, 0x74, 0x70, 0x3A, 0x2F, 0x2F, 0x65, 0x78, 0x61, 0x6D, 0x70, 0x6C, 0x65, 0x2E, 0x63, 0x6F, 0x6D, 0x2F, 0x62, 0x65, 0x6E, 0x63, 0x68, 0x6D, 0x61, 0x72, 0x6B, }; pub const cmd_start_bench: []const u8 = &.{ 0x1, 0x0, 0x0, 0x0, }; pub const cmd_stop_bench: []const u8 = &.{ 0x2, 0x0, 0x0, 0x0, }; pub const cmd_ack: []const u8 = &.{ 0x3, 0x0, 0x0, 0x0, }; pub const cmd_ping_perf: []const u8 = &.{ 0x4, 0x0, 0x0, 0x0, }; pub const cmd_set_integration: []const u8 = &.{ 0x5, 0x0, 0x0, 0x0, 0x10, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x74, 0x65, 0x73, 0x74, 0x2D, 0x69, 0x6E, 0x74, 0x65, 0x67, 0x72, 0x61, 0x74, 0x69, 0x6F, 0x6E, 0x5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x31, 0x2E, 0x30, 0x2E, 0x30, }; pub const cmd_err: []const u8 = &.{ 0x6, 0x0, 0x0, 0x0, }; pub const cmd_add_marker_sample_start: []const u8 = &.{ 0x7, 0x0, 0x0, 0x0, 0x39, 0x30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xE8, 0x3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; pub const cmd_add_marker_sample_end: []const u8 = &.{ 0x7, 0x0, 0x0, 0x0, 0x39, 0x30, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0xD0, 0x7, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; pub const cmd_add_marker_benchmark_start: []const u8 = &.{ 0x7, 0x0, 0x0, 0x0, 0x39, 0x30, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, 0xB8, 0xB, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; pub const cmd_add_marker_benchmark_end: []const u8 = &.{ 0x7, 0x0, 0x0, 0x0, 0x39, 0x30, 0x0, 0x0, 0x3, 0x0, 0x0, 0x0, 0xA0, 0xF, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; pub const cmd_set_version: []const u8 = &.{ 0x8, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; pub const cmd_get_runner_mode: []const u8 = &.{ 0x9, 0x0, 0x0, 0x0, }; pub const cmd_runner_mode_perf: []const u8 = &.{ 0xA, 0x0, 0x0, 0x0, 0x1, 0x0, 0x0, 0x0, }; pub const cmd_runner_mode_simulation: []const u8 = &.{ 0xA, 0x0, 0x0, 0x0, 0x2, 0x0, 0x0, 0x0, }; pub const cmd_runner_mode_analysis: []const u8 = &.{ 0xA, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, }; codspeed-4.4.1/instrument-hooks/src/tests/runner.zig000064400000000000000000000014521046102023000207300ustar 00000000000000const std = @import("std"); const builtin = @import("builtin"); pub fn main() !void { const out = std.io.getStdOut().writer(); for (builtin.test_functions) |t| { std.testing.allocator_instance = .{}; const name = extractName(t); const result = t.func(); if (result) |_| { try std.fmt.format(out, "[SUCCESS] {s}\n", .{name}); } else |err| { try std.fmt.format(out, "[FAIL] {s}: {}\n", .{ t.name, err }); } if (std.testing.allocator_instance.deinit() == .leak) { try std.fmt.format(out, "{s} leaked memory\n", .{name}); } } } fn extractName(t: std.builtin.TestFn) []const u8 { const marker = std.mem.lastIndexOf(u8, t.name, ".test.") orelse return t.name; return t.name[marker + 6 ..]; } codspeed-4.4.1/instrument-hooks/src/utils.zig000064400000000000000000000066351046102023000174250ustar 00000000000000const std = @import("std"); const c = @cImport(@cInclude("time.h")); const errno = @cImport(@cInclude("errno.h")); const fcntl_h = @cImport(@cInclude("fcntl.h")); extern "c" fn nanosleep(nanos: u64) c_int; extern "c" fn printf(format: [*c]const c_char, ...) c_int; // Note: Using printf to avoid the extra code from std.log/std.debug. Those won't // compile because they are internally using syscalls (for Mutexes) which aren't cross-platform. // // This wrapper converts Zig string literals to null-terminated c_char arrays and handles // variadic argument forwarding to the C printf function. pub fn print(comptime fmt: []const u8, args: anytype) void { // Create a comptime null-terminated c_char array from the format string const fmt_z = comptime blk: { var buf: [fmt.len:0]c_char = undefined; for (fmt, 0..) |byte, i| { buf[i] = byte; } buf[fmt.len] = 0; break :blk buf; }; _ = @call(.auto, printf, .{&fmt_z} ++ args); } // Adaptation of [`std.Thread.sleep`](https://ziglang.org/documentation/0.14.0/std/#std.Thread.sleep) // to ensure that the C code is architecture-independent. The stdlib implementation uses inline syscalls, // which only works on a single architecture and is not portable. pub fn sleep(nanoseconds: u64) void { const s = nanoseconds / std.time.ns_per_s; const ns = nanoseconds % std.time.ns_per_s; var req: c.struct_timespec = .{ .tv_sec = std.math.cast(c.time_t, s) orelse std.math.maxInt(c.time_t), .tv_nsec = std.math.cast(c_long, ns) orelse std.math.maxInt(c_long), }; var rem: c.struct_timespec = undefined; while (true) { const ret = c.nanosleep(&req, &rem); if (ret == errno.EINTR) { req = rem; continue; } else { return; } } } pub fn setNonBlocking(fd: std.posix.fd_t) void { const current_flags = fcntl_h.fcntl(fd, fcntl_h.F_GETFL, @as(c_int, 0)); const new_flags = current_flags | fcntl_h.O_NONBLOCK; _ = fcntl_h.fcntl(fd, fcntl_h.F_SETFL, new_flags); } // Returns monotonic time since boot in nanoseconds. // // NOTE: Maximum representable timestamp is u64::MAX nanoseconds = 18,446,744,073,709,551,615 ns // which equals ~584.94 years from epoch (year 2554-07-21T23:34:33.709551615). Since CLOCK_MONOTONIC // measures time since boot, a system would need to run for ~585 years continuously to overflow this value. pub fn clock_gettime_monotonic() u64 { const ts = std.posix.clock_gettime(std.posix.clockid_t.MONOTONIC) catch unreachable; const s = @as(u64, @intCast(ts.sec)) * std.time.ns_per_s; const nsec: u64 = @intCast(ts.nsec); return s + nsec; } test "sleep for at least 1 second" { const start = try std.time.Instant.now(); sleep(1 * std.time.ns_per_s); const end = try std.time.Instant.now(); const elapsed_ns: u64 = end.since(start); const elapsed_s = elapsed_ns / std.time.ns_per_s; std.debug.assert(elapsed_s >= 1); std.debug.assert(elapsed_s < 2); } test "print function works without crashing" { // Test with no arguments print("Hello, World!\n", .{}); // Test with string argument print("Hello, %s!\n", .{"Zig"}); // Test with multiple arguments print("Number: %d, String: %s\n", .{ @as(c_int, 42), "test" }); // Test with format specifiers that need proper types print("Precision test: %.*s\n", .{ @as(c_int, 5), "HelloWorld" }); } codspeed-4.4.1/src/codspeed.rs000064400000000000000000000104171046102023000143460ustar 00000000000000use crate::measurement; use colored::Colorize; use std::ffi::CString; pub use std::hint::black_box; pub const WARMUP_RUNS: u32 = 5; pub fn display_native_harness() { eprintln!("Harness: codspeed v{}", env!("CARGO_PKG_VERSION"),); } #[derive(PartialEq)] pub enum InstrumentationStatus { /// Instrumentation detected via inline assembly directly Valgrind, /// Instrumentation detected via InstrumentHooks InstrumentHooks(&'static crate::instrument_hooks::InstrumentHooks), NotInstrumented, } impl InstrumentationStatus { pub fn is_instrumented(&self) -> bool { *self != InstrumentationStatus::NotInstrumented } } pub struct CodSpeed { benchmarked: Vec, current_benchmark: CString, group_stack: Vec, instrumentation_status: InstrumentationStatus, } impl CodSpeed { pub fn new() -> Self { use crate::instrument_hooks::InstrumentHooks; let instrumentation_status = { // We completely bypass InstrumentHooks if we detect Valgrind via inline assembly // Until we can reliably get rid of the inline assembly without causing breaking // changes in CPU simulation measurements by switching to InstrumentHooks only, we need // to keep this separation. if measurement::is_instrumented() { InstrumentationStatus::Valgrind } else { let hooks_instance = InstrumentHooks::instance(); if hooks_instance.is_instrumented() { InstrumentationStatus::InstrumentHooks(hooks_instance) } else { InstrumentationStatus::NotInstrumented } } }; if !instrumentation_status.is_instrumented() { eprintln!( "{} codspeed is enabled, but no performance measurement will be made since it's running in an unknown environment.", "NOTICE:".to_string().bold() ); }; measurement::set_metadata(); Self { benchmarked: Vec::new(), current_benchmark: CString::new("").expect("CString::new failed"), group_stack: Vec::new(), instrumentation_status, } } pub fn push_group(&mut self, group: &str) { self.group_stack.push(group.to_string()); } pub fn pop_group(&mut self) { self.group_stack.pop(); } #[inline(always)] pub fn start_benchmark(&mut self, name: &str) { self.current_benchmark = CString::new(name).expect("CString::new failed"); if let InstrumentationStatus::InstrumentHooks(hooks_instance) = &self.instrumentation_status { let _ = hooks_instance.start_benchmark(); } // We intentionally do this no matter the instrumentation status. // The overhead in case we are not running valgrind is extremely low, and we do not want to // add a conditionnal branch to valgrind measurements. measurement::start(); } #[inline(always)] pub fn end_benchmark(&mut self) { // We intentionally do this no matter the instrumentation status. // The overhead in case we are not running valgrind is extremely low, and we do not want to // add a conditionnal branch to valgrind measurements. measurement::stop(&self.current_benchmark); if let InstrumentationStatus::InstrumentHooks(hooks_instance) = &self.instrumentation_status { let _ = hooks_instance.stop_benchmark(); let _ = hooks_instance.set_executed_benchmark(&self.current_benchmark.to_string_lossy()); } self.benchmarked .push(self.current_benchmark.to_str().unwrap().to_string()); let action_str = if self.instrumentation_status.is_instrumented() { "Measured" } else { "Checked" }; let group_str = if self.group_stack.is_empty() { "".to_string() } else { format!(" (group: {})", self.group_stack.join("/")) }; println!( "{}: {}{}", action_str, self.current_benchmark.to_string_lossy(), group_str ); } } impl Default for CodSpeed { fn default() -> Self { Self::new() } } codspeed-4.4.1/src/instrument_hooks/bindings.rs000064400000000000000000000033701046102023000177700ustar 00000000000000/* automatically generated by rust-bindgen 0.72.1 */ pub const MARKER_TYPE_SAMPLE_START: u32 = 0; pub const MARKER_TYPE_SAMPLE_END: u32 = 1; pub const MARKER_TYPE_BENCHMARK_START: u32 = 2; pub const MARKER_TYPE_BENCHMARK_END: u32 = 3; pub type InstrumentHooks = *mut u64; extern "C" { pub fn instrument_hooks_init() -> *mut InstrumentHooks; } extern "C" { pub fn instrument_hooks_deinit(arg1: *mut InstrumentHooks); } extern "C" { pub fn instrument_hooks_is_instrumented(arg1: *mut InstrumentHooks) -> bool; } extern "C" { pub fn instrument_hooks_start_benchmark(arg1: *mut InstrumentHooks) -> u8; } extern "C" { pub fn instrument_hooks_stop_benchmark(arg1: *mut InstrumentHooks) -> u8; } extern "C" { pub fn instrument_hooks_set_executed_benchmark( arg1: *mut InstrumentHooks, pid: i32, uri: *const ::std::os::raw::c_char, ) -> u8; } extern "C" { pub fn instrument_hooks_executed_benchmark( arg1: *mut InstrumentHooks, pid: i32, uri: *const ::std::os::raw::c_char, ) -> u8; } extern "C" { pub fn instrument_hooks_set_integration( arg1: *mut InstrumentHooks, name: *const ::std::os::raw::c_char, version: *const ::std::os::raw::c_char, ) -> u8; } extern "C" { pub fn instrument_hooks_add_marker( arg1: *mut InstrumentHooks, pid: u32, marker_type: u8, timestamp: u64, ) -> u8; } extern "C" { pub fn instrument_hooks_current_timestamp() -> u64; } pub const instrument_hooks_feature_t_FEATURE_DISABLE_CALLGRIND_MARKERS: instrument_hooks_feature_t = 0; pub type instrument_hooks_feature_t = ::std::os::raw::c_uint; extern "C" { pub fn instrument_hooks_set_feature(feature: instrument_hooks_feature_t, enabled: bool); } codspeed-4.4.1/src/instrument_hooks/ffi.rs000064400000000000000000000005211046102023000167320ustar 00000000000000#![allow(non_upper_case_globals)] #![allow(non_camel_case_types)] #![allow(non_snake_case)] #![allow(unused)] // Use pre-generated bindings instead of generating at build time so that downstream // users don't need to install `libclang`. // // To regenerate bindings, run: // ``` // ./update-bindings.sh // ``` include!("bindings.rs"); codspeed-4.4.1/src/instrument_hooks/mod.rs000064400000000000000000000142351046102023000167540ustar 00000000000000#[cfg(use_instrument_hooks)] mod ffi; #[cfg(use_instrument_hooks)] mod linux_impl { use super::ffi; use std::ffi::CString; use std::sync::OnceLock; #[derive(PartialEq)] pub struct InstrumentHooks(*mut ffi::InstrumentHooks); unsafe impl Send for InstrumentHooks {} unsafe impl Sync for InstrumentHooks {} impl InstrumentHooks { #[inline(always)] pub fn new() -> Option { let ptr = unsafe { ffi::instrument_hooks_init() }; if ptr.is_null() { None } else { Some(InstrumentHooks(ptr)) } } /// Returns a singleton instance of `InstrumentHooks`. #[inline(always)] pub fn instance() -> &'static Self { static INSTANCE: OnceLock = OnceLock::new(); INSTANCE.get_or_init(|| { let instance = InstrumentHooks::new().expect("Failed to initialize InstrumentHooks"); instance .set_integration("codspeed-rust", env!("CARGO_PKG_VERSION")) .expect("Failed to set integration"); instance }) } #[inline(always)] pub fn is_instrumented(&self) -> bool { unsafe { ffi::instrument_hooks_is_instrumented(self.0) } } #[inline(always)] pub fn start_benchmark(&self) -> Result<(), u8> { let result = unsafe { ffi::instrument_hooks_start_benchmark(self.0) }; if result == 0 { Ok(()) } else { Err(result) } } #[inline(always)] pub fn stop_benchmark(&self) -> Result<(), u8> { let result = unsafe { ffi::instrument_hooks_stop_benchmark(self.0) }; if result == 0 { Ok(()) } else { Err(result) } } #[inline(always)] pub fn set_executed_benchmark(&self, uri: &str) -> Result<(), u8> { let pid = std::process::id() as i32; let c_uri = CString::new(uri).map_err(|_| 1u8)?; let result = unsafe { ffi::instrument_hooks_set_executed_benchmark(self.0, pid, c_uri.as_ptr()) }; if result == 0 { Ok(()) } else { Err(result) } } #[inline(always)] pub fn set_integration(&self, name: &str, version: &str) -> Result<(), u8> { let c_name = CString::new(name).map_err(|_| 1u8)?; let c_version = CString::new(version).map_err(|_| 1u8)?; let result = unsafe { ffi::instrument_hooks_set_integration(self.0, c_name.as_ptr(), c_version.as_ptr()) }; if result == 0 { Ok(()) } else { Err(result) } } #[inline(always)] pub fn add_benchmark_timestamps(&self, start: u64, end: u64) { let pid = std::process::id(); unsafe { ffi::instrument_hooks_add_marker( self.0, pid, ffi::MARKER_TYPE_BENCHMARK_START as u8, start, ) }; unsafe { ffi::instrument_hooks_add_marker( self.0, pid, ffi::MARKER_TYPE_BENCHMARK_END as u8, end, ) }; } #[inline(always)] pub fn current_timestamp() -> u64 { #[cfg(target_os = "linux")] { use nix::sys::time::TimeValLike; nix::time::clock_gettime(nix::time::ClockId::CLOCK_MONOTONIC) .expect("Failed to get current time") .num_nanoseconds() as u64 } #[cfg(not(target_os = "linux"))] unsafe { ffi::instrument_hooks_current_timestamp() } } pub fn disable_callgrind_markers() { unsafe { ffi::instrument_hooks_set_feature( ffi::instrument_hooks_feature_t_FEATURE_DISABLE_CALLGRIND_MARKERS, true, ) }; } } impl Drop for InstrumentHooks { fn drop(&mut self) { if !self.0.is_null() { unsafe { ffi::instrument_hooks_deinit(self.0) }; } } } } #[cfg(not(use_instrument_hooks))] mod other_impl { #[derive(PartialEq)] pub struct InstrumentHooks; impl InstrumentHooks { pub fn instance() -> &'static Self { static INSTANCE: InstrumentHooks = InstrumentHooks; &INSTANCE } pub fn is_instrumented(&self) -> bool { false } pub fn start_benchmark(&self) -> Result<(), u8> { Ok(()) } pub fn stop_benchmark(&self) -> Result<(), u8> { Ok(()) } pub fn set_executed_benchmark(&self, _uri: &str) -> Result<(), u8> { Ok(()) } pub fn set_integration(&self, _name: &str, _version: &str) -> Result<(), u8> { Ok(()) } pub fn add_benchmark_timestamps(&self, _start: u64, _end: u64) {} pub fn current_timestamp() -> u64 { 0 } pub fn disable_callgrind_markers() {} } } #[cfg(use_instrument_hooks)] pub use linux_impl::InstrumentHooks; #[cfg(not(use_instrument_hooks))] pub use other_impl::InstrumentHooks; #[cfg(test)] mod tests { use super::InstrumentHooks; #[test] fn test_instrument_hooks() { let hooks = InstrumentHooks::instance(); assert!(!hooks.is_instrumented() || hooks.start_benchmark().is_ok()); assert!(hooks.set_executed_benchmark("test_uri").is_ok()); assert!(hooks.set_integration("test_integration", "1.0.0").is_ok()); let start = InstrumentHooks::current_timestamp(); let end = start + 1_000_000; // Simulate 1ms later hooks.add_benchmark_timestamps(start, end); assert!(!hooks.is_instrumented() || hooks.stop_benchmark().is_ok()); } } codspeed-4.4.1/src/instrument_hooks/update-bindings.sh000075500000000000000000000003241046102023000212350ustar 00000000000000#!/usr/bin/env bash set -euo pipefail bindgen ../../instrument-hooks/includes/core.h \ -o bindings.rs \ --rust-target 1.74 \ --allowlist-function "instrument_hooks_.*" \ --allowlist-var "MARKER_TYPE_.*" codspeed-4.4.1/src/lib.rs000064400000000000000000000002011046102023000133140ustar 00000000000000pub mod codspeed; pub mod instrument_hooks; mod macros; mod measurement; mod request; pub mod utils; pub mod walltime_results; codspeed-4.4.1/src/macros.rs000064400000000000000000000023771046102023000140520ustar 00000000000000#[macro_export] macro_rules! abs_file { () => { std::path::PathBuf::from( std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT") .expect("Could not find CODSPEED_CARGO_WORKSPACE_ROOT env variable, make sure you are using the latest version of cargo-codspeed") ) .join(file!()) .to_string_lossy() }; } #[macro_export] macro_rules! codspeed_uri { ( $name:ident ) => { format!("{}::{}", file!(), stringify!($name)) }; ( $function:path ) => { format!("{}::{}", file!(), stringify!($function)) }; } #[macro_export] macro_rules! codspeed_bench { ( $name:ident, $bench_payload:expr) => { pub fn $name(codspeed: &mut $crate::codspeed::CodSpeed) { let uri = codspeed::codspeed_uri!($name); codspeed.start_benchmark(uri.as_str()); $crate::codspeed::black_box($bench_payload()); codspeed.end_benchmark(); } }; } #[macro_export] macro_rules! codspeed_main { ( $( $target:path ),+ $(,)* ) => { fn main() { $crate::codspeed::display_native_harness(); let mut codspeed = $crate::codspeed::CodSpeed::new(); $( $target(&mut codspeed); )+ } } } codspeed-4.4.1/src/measurement.rs000064400000000000000000000030361046102023000151040ustar 00000000000000use std::ffi::CString; use crate::request::{send_client_request, ClientRequest, Value}; #[inline(always)] pub fn is_instrumented() -> bool { let valgrind_depth = unsafe { send_client_request( 0, &[ClientRequest::RunningOnValgrind as Value, 0, 0, 0, 0, 0], ) }; valgrind_depth > 0 } #[inline(always)] pub fn set_metadata() { let full_metadata = CString::new(format!( "Metadata: codspeed-rust {}", env!("CARGO_PKG_VERSION") )) .expect("CString::new failed"); unsafe { send_client_request( 0, &[ ClientRequest::DumpStatisticsAt as Value, full_metadata.as_ptr() as Value, 0, 0, 0, 0, ], ); } } #[inline(always)] pub fn start() { unsafe { send_client_request(0, &[ClientRequest::ZeroStatistics as Value, 0, 0, 0, 0, 0]); send_client_request( 0, &[ClientRequest::StartInstrumentation as Value, 0, 0, 0, 0, 0], ); } } #[inline(always)] pub fn stop(name: &CString) { unsafe { send_client_request( 0, &[ClientRequest::StopInstrumentation as Value, 0, 0, 0, 0, 0], ); send_client_request( 0, &[ ClientRequest::DumpStatisticsAt as Value, name.as_ptr() as Value, 0, 0, 0, 0, ], ); } } codspeed-4.4.1/src/request/arch/aarch64.rs000064400000000000000000000006271046102023000164170ustar 00000000000000use std::arch::asm; pub type Value = u64; #[inline(always)] pub unsafe fn send_client_request(default: Value, args: &[Value; 6]) -> Value { let mut value = default; asm!( "ror x12, x12, #3", "ror x12, x12, #13", "ror x12, x12, #51", "ror x12, x12, #61", "orr x10, x10, x10", in("x4") args.as_ptr(), inlateout("x3") value, ); value } codspeed-4.4.1/src/request/arch/arm.rs000064400000000000000000000006471046102023000157500ustar 00000000000000use std::arch::asm; pub type Value = u32; #[inline(always)] pub unsafe fn send_client_request(default: Value, args: &[Value; 6]) -> Value { let mut value = default; asm!( "mov r12, r12, ror #3", "mov r12, r12, ror #13", "mov r12, r12, ror #29", "mov r12, r12, ror #19", "orr r10, r10, r10", in("r4") args.as_ptr(), inlateout("r3") value, ); value } codspeed-4.4.1/src/request/arch/unsupported.rs000064400000000000000000000002621046102023000175520ustar 00000000000000pub type Value = u32; #[inline(always)] pub unsafe fn send_client_request(_default: Value, _args: &[Value; 6]) -> Value { panic!("Not implemented for this architecture"); } codspeed-4.4.1/src/request/arch/x86.rs000064400000000000000000000006051046102023000156100ustar 00000000000000use std::arch::asm; pub type Value = u32; #[inline(always)] pub unsafe fn send_client_request(default: Value, args: &[Value; 6]) -> Value { let mut value = default; asm!( "rol edi, $$3", "rol edi, $$13", "rol edi, $$29", "rol edi, $$19", "xchg ebx, ebx", in("eax") args.as_ptr(), inlateout("edx") value, ); value } codspeed-4.4.1/src/request/arch/x86_64.rs000064400000000000000000000005751046102023000161270ustar 00000000000000use std::arch::asm; pub type Value = u64; #[inline(always)] pub unsafe fn send_client_request(default: Value, args: &[Value; 6]) -> Value { let mut value = default; asm!( "rol rdi, 3", "rol rdi, 13", "rol rdi, 61", "rol rdi, 51", "xchg rbx, rbx", in("rax") args.as_ptr(), inlateout("rdx") value, ); value } codspeed-4.4.1/src/request/mod.rs000064400000000000000000000014471046102023000150320ustar 00000000000000const CG_BASE: u32 = ((b'C' as u32) << 24) + ((b'T' as u32) << 16); #[allow(non_camel_case_types)] #[repr(u32)] pub enum ClientRequest { RunningOnValgrind = 0x1001, ZeroStatistics = CG_BASE + 1, DumpStatisticsAt = CG_BASE + 3, StartInstrumentation = CG_BASE + 4, StopInstrumentation = CG_BASE + 5, } pub use self::arch::{send_client_request, Value}; #[cfg(target_arch = "x86_64")] #[path = "arch/x86_64.rs"] mod arch; #[cfg(target_arch = "x86")] #[path = "arch/x86.rs"] mod arch; #[cfg(target_arch = "arm")] #[path = "arch/arm.rs"] mod arch; #[cfg(target_arch = "aarch64")] #[path = "arch/aarch64.rs"] mod arch; #[cfg(not(any( target_arch = "x86_64", target_arch = "x86", target_arch = "arm", target_arch = "aarch64" )))] #[path = "arch/unsupported.rs"] mod arch; codspeed-4.4.1/src/utils.rs000064400000000000000000000077711046102023000137310ustar 00000000000000use std::io; use std::path::{Path, PathBuf}; fn get_parent_git_repo_path(abs_path: &Path) -> io::Result { if abs_path.join(".git").exists() { Ok(abs_path.to_path_buf()) } else { get_parent_git_repo_path( abs_path .parent() .ok_or(io::Error::from(io::ErrorKind::NotFound))?, ) } } pub fn get_git_relative_path

(abs_path: P) -> PathBuf where P: AsRef, { if let Ok(canonicalized_abs_path) = abs_path.as_ref().canonicalize() { // `repo_path` is still canonicalized as it is a subpath of `canonicalized_abs_path` if let Ok(repo_path) = get_parent_git_repo_path(&canonicalized_abs_path) { canonicalized_abs_path .strip_prefix(repo_path) .expect("Repository path is malformed.") .to_path_buf() } else { canonicalized_abs_path } } else { abs_path.as_ref().to_path_buf() } } /// Fixes spaces around `::` created by stringify!($function). pub fn get_formated_function_path(function_path: impl Into) -> String { let function_path = function_path.into(); function_path.replace(" :: ", "::") } pub fn running_with_codspeed_runner() -> bool { std::env::var("CODSPEED_ENV").is_ok() } pub fn is_perf_enabled() -> bool { std::env::var("CODSPEED_PERF_ENABLED").is_ok() } /// Generate a statistically unique ID in a format resembling UUID v4. pub fn generate_unique_id() -> String { // Generate random bytes for UUID v4 let mut bytes = [0u8; 16]; getrandom::getrandom(&mut bytes).expect("Failed to generate random bytes"); // Extract values from bytes let r1 = u32::from_be_bytes([bytes[0], bytes[1], bytes[2], bytes[3]]); let r2 = u16::from_be_bytes([bytes[4], bytes[5]]); let r3 = u16::from_be_bytes([bytes[6], bytes[7]]); let r4 = u16::from_be_bytes([bytes[8], bytes[9]]); let r5 = u32::from_be_bytes([bytes[10], bytes[11], bytes[12], bytes[13]]); let r6 = u16::from_be_bytes([bytes[14], bytes[15]]); // Set version (4) and variant bits according to UUID v4 spec let r3_v4 = (r3 & 0x0fff) | 0x4000; // Version 4 let r4_variant = (r4 & 0x3fff) | 0x8000; // Variant 10 // Format as standard UUID: xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx // where y is one of 8, 9, A, or B format!("{r1:08x}-{r2:04x}-{r3_v4:04x}-{r4_variant:04x}-{r5:08x}{r6:04x}") } #[cfg(test)] mod tests { use super::*; use std::fs; use tempfile::tempdir; #[test] fn test_get_git_relative_path_found() { // Create a temp directory. let dir = tempdir().unwrap(); let git_dir = dir.path().join(".git"); fs::create_dir(git_dir).unwrap(); let nested_dir = dir.path().join("folder").join("nested_folder"); fs::create_dir_all(&nested_dir).unwrap(); let relative_path = get_git_relative_path(&nested_dir); assert_eq!(relative_path, PathBuf::from("folder/nested_folder")); } #[test] fn test_get_git_relative_path_not_found() { let dir = tempdir().unwrap(); let path_dir = dir.path().join("folder"); fs::create_dir_all(&path_dir).unwrap(); let relative_path = get_git_relative_path(&path_dir); assert_eq!(relative_path, path_dir.canonicalize().unwrap()); } #[cfg(unix)] #[test] fn test_get_git_relative_path_not_found_with_symlink() { let dir = tempdir().unwrap(); let path_dir = dir.path().join("folder"); fs::create_dir_all(&path_dir).unwrap(); let symlink = dir.path().join("symlink"); std::os::unix::fs::symlink(&path_dir, &symlink).unwrap(); let relative_path = get_git_relative_path(&symlink); assert_eq!(relative_path, symlink.canonicalize().unwrap()); } #[test] fn test_get_formated_function_path() { let input = "std :: vec :: Vec :: new"; let expected_output = "std::vec::Vec::new".to_string(); assert_eq!(get_formated_function_path(input), expected_output); } } codspeed-4.4.1/src/walltime_results.rs000064400000000000000000000236101046102023000161560ustar 00000000000000use anyhow::{Context, Result}; use std::{ io::Write, path::{Path, PathBuf}, }; use serde::{Deserialize, Serialize}; use statrs::statistics::{Data, Distribution, Max, Min, OrderStatistics}; use crate::utils::generate_unique_id; const IQR_OUTLIER_FACTOR: f64 = 1.5; const STDEV_OUTLIER_FACTOR: f64 = 3.0; #[derive(Debug, Serialize, Deserialize)] pub struct BenchmarkMetadata { pub name: String, pub uri: String, } #[derive(Debug, Serialize, Deserialize)] struct BenchmarkStats { min_ns: f64, max_ns: f64, mean_ns: f64, stdev_ns: f64, q1_ns: f64, median_ns: f64, q3_ns: f64, rounds: u64, total_time: f64, iqr_outlier_rounds: u64, stdev_outlier_rounds: u64, iter_per_round: u64, warmup_iters: u64, } #[derive(Debug, Serialize, Deserialize, Default)] struct BenchmarkConfig { warmup_time_ns: Option, min_round_time_ns: Option, max_time_ns: Option, max_rounds: Option, } #[derive(Debug, Serialize, Deserialize)] pub struct WalltimeBenchmark { #[serde(flatten)] metadata: BenchmarkMetadata, config: BenchmarkConfig, stats: BenchmarkStats, } impl WalltimeBenchmark { /// Entry point called in patched integration to harvest raw walltime data /// /// `CODSPEED_CARGO_WORKSPACE_ROOT` is expected to be set for this to work /// /// # Arguments /// /// - `scope`: The used integration, e.g. "divan" or "criterion" /// - `name`: The name of the benchmark /// - `uri`: The URI of the benchmark /// - `iters_per_round`: The number of iterations for each round (=sample_size), e.g. `[1, 2, 3]` (variable) or `[2, 2, 2, 2]` (constant). /// - `times_per_round_ns`: The measured time for each round in nanoseconds, e.g. `[1000, 2000, 3000]` /// - `max_time_ns`: The time limit for the benchmark in nanoseconds (if defined) /// /// # Pseudo-code /// /// ```text /// let sample_count = /* The number of executions for the same benchmark. */ /// let sample_size = iters_per_round = vec![/* The number of iterations within each sample. */]; /// for round in 0..sample_count { /// let times_per_round_ns = 0; /// for iteration in 0..sample_size[round] { /// run_benchmark(); /// times_per_round_ns += /* measured execution time */; /// } /// } /// ``` /// pub fn collect_raw_walltime_results( scope: &str, name: String, uri: String, iters_per_round: Vec, times_per_round_ns: Vec, max_time_ns: Option, ) { if !crate::utils::running_with_codspeed_runner() { return; } let workspace_root = std::env::var("CODSPEED_CARGO_WORKSPACE_ROOT").map(PathBuf::from); let Ok(workspace_root) = workspace_root else { eprintln!("codspeed failed to get workspace root. skipping"); return; }; let data = WalltimeBenchmark::from_runtime_data( name, uri, iters_per_round, times_per_round_ns, max_time_ns, ); data.dump_to_results(&workspace_root, scope); } pub fn from_runtime_data( name: String, uri: String, iters_per_round: Vec, times_per_round_ns: Vec, max_time_ns: Option, ) -> Self { let total_time = times_per_round_ns.iter().sum::() as f64 / 1_000_000_000.0; let time_per_iteration_per_round_ns: Vec<_> = times_per_round_ns .into_iter() .zip(&iters_per_round) .map(|(time_per_round, iter_per_round)| time_per_round / iter_per_round) .map(|t| t as f64) .collect::>(); let mut data = Data::new(time_per_iteration_per_round_ns); let rounds = data.len() as u64; let mean_ns = data.mean().unwrap(); let stdev_ns = if data.len() < 2 { // std_dev() returns f64::NAN if data has less than two entries, so we have to // manually handle this case. 0.0 } else { data.std_dev().unwrap() }; let q1_ns = data.quantile(0.25); let median_ns = data.median(); let q3_ns = data.quantile(0.75); let iqr_ns = q3_ns - q1_ns; let iqr_outlier_rounds = data .iter() .filter(|&&t| { t < q1_ns - IQR_OUTLIER_FACTOR * iqr_ns || t > q3_ns + IQR_OUTLIER_FACTOR * iqr_ns }) .count() as u64; let stdev_outlier_rounds = data .iter() .filter(|&&t| { t < mean_ns - STDEV_OUTLIER_FACTOR * stdev_ns || t > mean_ns + STDEV_OUTLIER_FACTOR * stdev_ns }) .count() as u64; let min_ns = data.min(); let max_ns = data.max(); // TODO(COD-1056): We currently only support single iteration count per round let iter_per_round = (iters_per_round.iter().sum::() / iters_per_round.len() as u128) as u64; let warmup_iters = 0; // FIXME: add warmup detection let stats = BenchmarkStats { min_ns, max_ns, mean_ns, stdev_ns, q1_ns, median_ns, q3_ns, rounds, total_time, iqr_outlier_rounds, stdev_outlier_rounds, iter_per_round, warmup_iters, }; WalltimeBenchmark { metadata: BenchmarkMetadata { name, uri }, config: BenchmarkConfig { max_time_ns: max_time_ns.map(|t| t as f64), ..Default::default() }, stats, } } fn dump_to_results(&self, workspace_root: &Path, scope: &str) { let output_dir = result_dir_from_workspace_root(workspace_root).join(scope); std::fs::create_dir_all(&output_dir).unwrap(); let bench_id = generate_unique_id(); let output_path = output_dir.join(format!("{bench_id}.json")); let mut writer = std::fs::File::create(&output_path).expect("Failed to create the file"); serde_json::to_writer_pretty(&mut writer, self).expect("Failed to write the data"); writer.flush().expect("Failed to flush the writer"); } pub fn is_invalid(&self) -> bool { self.stats.min_ns < f64::EPSILON } pub fn name(&self) -> &str { &self.metadata.name } } #[derive(Debug, Serialize, Deserialize)] struct Instrument { #[serde(rename = "type")] type_: String, } #[derive(Debug, Serialize, Deserialize)] struct Creator { name: String, version: String, pid: u32, } #[derive(Debug, Serialize, Deserialize)] pub struct WalltimeResults { creator: Creator, instrument: Instrument, benchmarks: Vec, } impl WalltimeResults { pub fn collect_walltime_results(workspace_root: &Path) -> Result { // retrieve data from `{workspace_root}/target/codspeed/raw_results/{scope}/*.json let benchmarks = glob::glob(&format!( "{}/**/*.json", result_dir_from_workspace_root(workspace_root) .to_str() .unwrap(), ))? .map(|sample| -> Result<_> { let sample = sample?; serde_json::from_reader::<_, WalltimeBenchmark>(std::fs::File::open(&sample)?) .context("Failed to read benchmark data") }) .collect::>>()?; Ok(WalltimeResults { instrument: Instrument { type_: "walltime".to_string(), }, creator: Creator { name: "codspeed-rust".to_string(), version: env!("CARGO_PKG_VERSION").to_string(), pid: std::process::id(), }, benchmarks, }) } pub fn clear(workspace_root: &Path) -> Result<()> { let raw_results_dir = result_dir_from_workspace_root(workspace_root); std::fs::remove_dir_all(&raw_results_dir).ok(); // ignore errors when the directory does not exist std::fs::create_dir_all(&raw_results_dir) .context("Failed to create raw_results directory")?; Ok(()) } pub fn benchmarks(&self) -> &[WalltimeBenchmark] { &self.benchmarks } } // FIXME: This assumes that the cargo target dir is `target`, and duplicates information with // `cargo-codspeed::helpers::get_codspeed_target_dir` fn result_dir_from_workspace_root(workspace_root: &Path) -> PathBuf { workspace_root .join("target") .join("codspeed") .join("walltime") .join("raw_results") } #[cfg(test)] mod tests { use super::*; const NAME: &str = "benchmark"; const URI: &str = "test::benchmark"; #[test] fn test_parse_single_benchmark() { let benchmark = WalltimeBenchmark::from_runtime_data( NAME.to_string(), URI.to_string(), vec![1], vec![42], None, ); assert_eq!(benchmark.stats.stdev_ns, 0.); assert_eq!(benchmark.stats.min_ns, 42.); assert_eq!(benchmark.stats.max_ns, 42.); assert_eq!(benchmark.stats.mean_ns, 42.); } #[test] fn test_parse_bench_with_variable_iterations() { let iters_per_round = vec![1, 2, 3, 4, 5, 6]; let total_rounds = iters_per_round.iter().sum::() as f64; let benchmark = WalltimeBenchmark::from_runtime_data( NAME.to_string(), URI.to_string(), iters_per_round, vec![42, 42 * 2, 42 * 3, 42 * 4, 42 * 5, 42 * 6], None, ); assert_eq!(benchmark.stats.stdev_ns, 0.); assert_eq!(benchmark.stats.min_ns, 42.); assert_eq!(benchmark.stats.max_ns, 42.); assert_eq!(benchmark.stats.mean_ns, 42.); assert_eq!( benchmark.stats.total_time, 42. * total_rounds / 1_000_000_000.0 ); } }